#!/usr/bin/python3
"""
Interface and Uploader to the lightmeter.

This script implements a polling readout of the lightmeter, a collection
of the data into files, the upload of these files to the GAVO data center,
and some informational web pages for the operators.

To allow running this as an ordinary user, you need the following udev
rule (e.g., in /etc/udev/rules.d/10-lightmeter.rules):

ATTR{idProduct}=="fcb7", ATTR{idVendor}=="04d8", SUBSYSTEM=="usb", MODE:="0666"

You'll also usually want an init script.  make install provides one
as embedded below.  That's mainly for debian sysv systems, but I'd
expect it to work on other systems, too.

For systemd, here's a tentative unit file::

	[Unit]
	Description=run lightmeter
	After=local-fs.target

	[Service]
	Type=simple
	ExecStart=/usr/local/bin/lightmeter.py foreground
	Restart=on-failure
	RestartSec=3600
	User=lightmeter

	[Install]
	WantedBy=multi-user.target

(this assumes you've put this file into /usr/local/bin/lightmeter.py and
made it executable)

Dependencies:
	* python3-libusb1
	* python3-usb1
	* python3-requests
	* python3-matplotlib if you want the plots

TODO:

* Behave sanely when the lightmeter isn't connected or broken (diagnostics,
  try again regularly)
* On an internal re-start, the web server is disconnected.  It needs
  to be restarted, too.
* Define a simple plug-in interface, good enough for watchdogs, LED control
* web-based configuration
"""

import atexit
import http.server
import calendar
import configparser
import collections
import datetime
import glob
import gzip
import http.client
import io
import logging
import logging.handlers
import math
import os
import pwd
import random
import re
import signal
import struct
import sys
import tempfile
import threading
import time
import traceback
import warnings

try:
	import matplotlib
	matplotlib.use("Agg")
	from matplotlib import pyplot
except ImportError:
	# no matplotlib, no plots, no problem
	pass

import requests
import usb1
import libusb1

VERSION = "0.4"


PROXY = None
# If you are forced to use a proxy to access the web, enter it here,
# like this:
#PROXY = "dwdmx5.dwd.de", 80

# the following is for the initscript sub-command
INITSCRIPT =\
"""#!/bin/sh -e
### BEGIN INIT INFO
# Provides:          lightmeter
# Required-Start:    $local_fs $remote_fs $network $syslog
# Required-Stop:     $local_fs $remote_fs $network $syslog
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: Start/stop Lightmeter driver
### END INIT INFO
#

LM_USER="lightmeter"

. /lib/lsb/init-functions

test -f /etc/default/rcS && . /etc/default/rcS

call_server() {
	su $LM_USER -c "/usr/local/bin/lightmeter.py $1"
}

case $1 in
        start)
                log_daemon_msg "Starting Lightmeter driver" "lightmeter.py"
                if call_server start; then
                        log_end_msg 0
                else
                        log_end_msg 1
            fi
        ;;
        stop)
                log_daemon_msg "Stopping Lightmeter driver" "lightmeter.py"
                if call_server stop; then
            log_end_msg 0
        else
            log_end_msg 1
        fi
        ;;
        *)
                log_success_msg "Usage: /etc/init.d/lightmeter {start|stop}"
                exit 1
        ;;
esac
"""

LOGGER = logging.getLogger('lightmeter')
LOGGER.setLevel(logging.INFO)
LOGGER.addHandler(logging.handlers.SysLogHandler(
	address='/dev/log', facility="daemon"))
#LOGGER.addHandler(logging.handlers.RotatingFileHandler(
#              "testlog"))

################ Data for the HTTP server

TIME_FORMAT = "%H:%M:%S %Y-%m-%d"

STYLE = """
html {
	background: #000066;
	font-family: sans;
}

body {
	background: #444466;
	max-width: 700px;
	margin-left: auto;
	margin-right: auto;
	text-align: left;
	padding: 1em;
	color: #ffddff;
}

div {
	background: #ddddff;
	color: black;
	padding: 1ex 0.5em 1ex 0.5em;
	margin: 1ex 0.4em 1ex 0.4em;
	border: 1pt solid grey;
}

dl {
	margin: 0px;
}

h1#maintitle {
	text-align: center;
}

div#timebox {
	float:right;
}

div#physbox {
	float:left;
}

div#imagebox {
	float: left;
}

div#commbox {
	float: right;
}

dd.physvalue {
	font-size: 250%;
	font-weight: bold;
}

p.remark {
	margin-top: 2pt;
	margin-bottom: 2pt;
}

.val_uncalibrated {
	color: grey;
}
"""

STANDARD_HEADER_MATERIAL = """
<link rel="stylesheet" type="text/css" href="style.css"/>
"""

ERROR_TEMPLATE = """<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
		"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<head>
<title>$(title)</title>
$^
</head>
<body>
<h1>$(title)</h1>
<p class="errmsg">$(msg)</p>
</body>
</html>
"""

ROOT_TEMPLATE = """<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
		"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<head>
<meta http-equiv="refresh" content="1800"/>
<title>$(title)</title>
$^
</head>
<body>
<h1 id="maintitle">$(stationid) Lightmeter</h1>
<div id="timebox">
<dl>
<dt>UTC on server</dt>
<dd>$(utcnow)</dd>
<dt>Local time on server</dt>
<dd>$(localtime)</dd>
</dl>
</div>

<div id="physbox">
<dl>
<dt>Last flux [W/m<sup>2</sup>] $(calibwarning)</dt>
<dd class="physvalue" class="val_$[calibstatus]">$(lastcount)</dd>
<dt>Last temperature (celsius)</dt>
<dd class="physvalue">$(lasttemperature)</dd>
</dl>
</div>


<p style="clear:both"/>
<div id="imagebox">
<img src="/current.png" alt="[Image: Plot]"/>
<p class="remark">(updated 10-minutely at most)</p>
</div>

<div id="commbox">
<dl>
<dt>Last upload (local time)</dt>
<dd>$(last_upload)</dd>
<dt>Files in upload queue:</dt>
<dd>$(no_queued)</dd>
<dt>Files archived:</dt>
<dd>$(no_archived)</dd>
<dt>Free space on log drive:</dt>
<dd>$(log_drive_space)</dd>
</dl>
<p><a href="/upload.report">Inspect last upload report</a></p>
</div>

<p style="clear:both"/>
</body>
</html>
"""


################ The actual program

class Error(Exception):
	"""base class for all errors we raise.

	The idea is that such errors should be reported to the user,
	whereas all other exceptions are indicators that something happened
	that the authors don't handle.
	"""
	pass


class UploadError(Error):
	pass


def parse_skyglow_file(file_name):
	"""returns a sequence of (unix timestamp, temperature, count) tuples
	from the skyglow fName.

	We're actually specializing on what we are generating ourselves.
	"""
	with open(file_name, encoding="utf-8") as f:
		for ln in f:
			if ln.startswith("#"):
				continue
			try:
				ut, ltd, temp, count = ln.split(";")
				yield (
					calendar.timegm(time.strptime(ut, "%Y-%m-%dT%H:%M:%S")),
						float(temp), float(count))
			except ValueError:
				sys.stderr.write("Bad line in archived file %s:'%s'\n"%(file_name, ln))

def get_last_data(file_manager, history_bytes=500000):
	"""returns history seed from archive data.
	"""
	last_data = file_manager.get_newest_files(history_bytes)
	recorder = ValueRecorder()
	for fName in last_data:
		for ts, temp, count in parse_skyglow_file(fName):
			recorder.feed(count, temp, at_time=ts)
	return recorder.buffer


################### the central classes

class Config(object):
	"""configuration items.

	This class knows how to fill itself from ini-style files and can serialize
	itself back there.

	By default, the stuff is read from ~/.mlumrc/written to.  You can
	change that by setting MLUMRC; the machinery needs to be able to
	read and write that location.

	There is a mechanism for turning the strings coming in into typed
	values using the _parseitem_name.  On saving, all values are just
	stringified.
	"""
	verbose = False

	BUILTIN = """
[general]
# The station id (mandatory!)
stationid =
# The station access key (mandatory!)
accesskey =
# latitude of the station, in decimal degrees (southern latitudes have
# a minus sign; mandatory!)
latitude = unconfigured
# longitude of the station east of Greenwich, in decimal degrees
# (i.e., eastern longitudes are positive, western negative; mandatory!)
longitude = unconfigured
# elevation of the station (in meters; manatory!)
elevation = unconfigured
# IANA timezone name of your *local* time zone; take a string from
# http://en.wikipedia.org/wiki/List_of_tz_database_time_zones
# (and make sure your computer is actually configured to use that
# time zone; mandatory)
timezone = unconfigured

# sample interval, in seconds
sampleInterval = 2
# upload interval, in days
uploadInterval = 1
# root directory for data
rootDir = ~/lightmeter-data
# The base URL of the upload service
uploadbase = http://dc.zah.uni-heidelberg.de/lightmeter/q/upload/custom
# The type of the device; this is always IYA Lightmeter possibly with a
# version (in case you know, it'd look like "(Mark 2.3)")
deviceType = IYA Lightmeter
# some real name, possibly affiliation (like B. Honeydew, Muppet Labs)
operatorName: a/a
# Some verbose location description, like "Roof of the Muppet Theatre,
# Broadway, New York, USA)
locationName: n/a
# How does your computer get its time?  This could be something like
# NTP, GPS, monthly by hand; choose whatever conveys your practice to
# the users of your data.
syncMethod: n/a
# change this if your device doesn't look straight up.  North Horizon
# is 90,0, east horizon is 90, 90
pointing: 0,0

# The four calibration parameters.  What's given here is
# generic values that may make for bizarre results.  To have your
# data calibrated, contact GAVO.
calibA: 150000
calibB: 0.01
calibC: 2e-9
calibD: 0.004
# Change this to true once you have an actual calibration for your device.
calibrated: False

# The port the built-in HTTP server is listening on
httpPort: 1082
# A (valid) system user to run as when started as root
user:lightmeter
"""

	def __init__(self):
		self.settings = {}
		self.rc_name = os.path.expanduser(os.environ.get("MLUMRC", "~/.mlumrc"))
		self._set_defaults()

	_parseitem_uploadinterval = _parseitem_sampleinterval = \
		_parseitem_httpport = int
	_parseitem_caliba = _parseitem_calibb = \
	_parseitem_calibc = _parseitem_calibd =  float
	_parseitem_rootdir = staticmethod(os.path.expanduser)

	def _parse_boolean_item(self, literal):
		if literal.lower()=="false":
			return False
		elif literal.lower()=="true":
			return True
		else:
			raise ValueError("Booleans must be either 'True' or 'False'")

	_parseitem_calibrated = _parse_boolean_item

	def _set_defaults(self):
		self._parse_from_file(io.StringIO(self.BUILTIN))

	def _parse_from_file(self, source):
		"""fills settings from the file instance source
		"""
		parser = configparser.ConfigParser()
		parser.read_file(source)

		if parser.has_section("general"):
			for function, setting in parser.items("general"):
				if hasattr(self, "_parseitem_"+function.lower()):
					try:
						self.settings[function] = getattr(
							self, "_parseitem_"+function)(setting)
					except Exception as msg:
						warnings.warn("Config item %s=%s malformed (%s), ignored"%(
							function, setting, msg))
				else:
					self.settings[function] = setting

	def load(self):
		if os.path.isfile(self.rc_name):
			with open(self.rc_name, encoding="utf-8") as f:
				self._parse_from_file(f)

	def save(self):
		new_config = configparser.ConfigParser()
		new_config.add_section("general")
		for key, value in self.settings.items():
			new_config.set("general", key, str(value))

		fd, temp_name = tempfile.mkstemp(dir=os.environ.get("HOME", "/tmp"))
		f = os.fdopen(fd, "w", encoding="utf-8")
		new_config.write(f)
		f.close()
		os.rename(temp_name, self.rc_name)

	def get(self, key):
		"""returns a (typed) setting key.

		Non-existing settings will give KeyErrors; thus, only use what
		there is in the defaults file.
		"""
		return self.settings[key]

	def set(self, key, value):
		"""sets a setting to to value.

		value must be type-true.
		"""
		self.settings[key] = value

	def get_upload_url(self):
		if not self.get("stationid") or not self.get("accesskey"):
			raise UploadError("stationid and/or accesskey is empty: Cannot upload")
		return "%s/%s/%s"%(
			self.get("uploadbase"),
			self.get("stationid"),
			self.get("accesskey"))

	def to_physical(self, count, temperature):
		"""returns the flux in W/m2 from a count.

		This uses the calibration given by the configuration; if that's not
		tailored to the device in use, values here may be severely off.

		This method is a bit wonky in that it replaces itself with a
		version with burnt-in calibration on the first call.  Premature
		optimization, perhaps, but fun.
		"""
		a, b, c, d = list(map(self.get, ["caliba", "calibb", "calibc", "calibd"]))
		def to_physical(count, temperature):
			return c*(b*(a*math.exp(count*(1+d*temperature)/a)-1)+count)
		self.to_physical = to_physical
		return to_physical(count, temperature)


CONFIG = Config()



class ValueRecorder(object):
	"""An averaging buffer for lightmeter measurements.

	The Lightmeter class feeds its values here; this is for online retrieval
	and plotting.  The idea is to keep values for the last 24 hours.

	To retrieve the data, just fetch (unix-time, counts) tuples from buffer.
	"""
	def __init__(self, init_vals=[]):
		self.buffer = collections.deque(init_vals, maxlen=24*60)
		self.bin_size = 60//CONFIG.get("sampleinterval") or 1
		self.accumulator = []

	def feed(self, value, temperature, at_time=None):
		self.accumulator.append(value)
		if len(self.accumulator)>=self.bin_size:
			if at_time is None:
				at_time = time.time()
			self.buffer.append((at_time,
				CONFIG.to_physical(
					sum(self.accumulator[:self.bin_size])/float(self.bin_size),
					temperature)))
			self.accumulator = []


class LightmeterBase(object):
	"""a common base class for all implementation of actual or fake devices.
	"""

	last_count, last_temperature = None, None

	def get_info_dict(self):
		return {
			"lastcount": "%7.4g"%(CONFIG.to_physical(self.last_count,
				self.last_temperature)),
			"lasttemperature": "%4.1f"%(self.last_temperature),
			"calibstatus": (not CONFIG.get("calibrated") and "uncalibrated"
				) or "calibrated",
			"calibwarning": (not CONFIG.get("calibrated") and "UNCALIBRATED") or "",
		}

	def __init__(self, init_data):
		self.recorder = ValueRecorder(init_data)


class FakeLightmeter(LightmeterBase):
	"""a blind data generator for development.

	You can feed the internal history using a
	(unix timestamp, count) iterable init_data.
	"""
	cur_temp = 20
	cur_flux = 200000.

	def generate_fake_data(self):
		fake_time = time.time()-3600
		for dt in range(60):
			yield fake_time, self.cur_flux
			fake_time += 60
			self.cur_flux += math.sin(dt/10.)*600

	def get_temperature(self):
		self.cur_temp += random.randint(-1, 1)
		self.last_temperature = self.cur_temp
		return self.cur_temp

	def get_light(self):
		self.cur_flux *= 0.9999
		self.last_count = self.cur_flux
		self.recorder.feed(self.cur_flux, self.cur_temp)
		return self.cur_flux


def _bytify(s):
	if isinstance(s, str):
		return s.encode("ascii")
	return s


class Lightmeter(LightmeterBase):
	"""A proxy for the lightmeter.

	The interface to the first lightmeter is being claimed when the
	object is constructed.

	You can seed the internal history using a
	(unix timestamp, count) iterable init_data.
	"""
	supported_devices = [
		(0x04d8, 0x000c),
		(0x04d8, 0xfcb7),]

	endpoint_in_id = 0x81
	endpoint_out_id = 0x01

	handle = None

	def __init__(self, init_data):
		LightmeterBase.__init__(self, init_data)
		self.context = usb1.USBContext()
		self.context.setDebug(usb1.LOG_LEVEL_DEBUG)
		for id_vendor, id_product in self.supported_devices:
			self.handle = self.context.openByVendorIDAndProductID(
				vendor_id=id_vendor, product_id=id_product)
			if self.handle is not None:
				break
		else:
			raise libusb1.USBError("No lightmeter found")
		self.handle.claimInterface(0)

	def send(self, data, timeout=1000):
		data = _bytify(data)
		assert self.handle.bulkWrite(self.endpoint_out_id,
			data, timeout=timeout)==len(data)

	def recv(self, num_bytes, timeout=1000):
		return self.handle.bulkRead(self.endpoint_in_id,
			num_bytes, timeout=timeout)

	def get_temperature(self):
		"""returns the current device temperature in degrees centigrade.
		"""
		self.send("T")
		raw = struct.unpack("<h", self.recv(2))[0]
		self.last_temperature = raw/128.
		return self.last_temperature

	factors_for_range = {
		1: 120,# daylight
		2: 8,
		3: 4,
		4: 2,
		5: 1, # weakest sky glow
	}

	def get_light(self):
		"""returns the current flux in device units.
		"""
		self.send("L")
		raw_response = list(self.recv(7)) # FIXME: old devs return only 3 bytes
		count = (raw_response[0]+(raw_response[1]<<8)
			)*self.factors_for_range[raw_response[2]]

		self.last_count = count

		# don't feed history unless we have a temperature
		if self.last_temperature is not None:
			self.recorder.feed(count, self.last_temperature)
		return count


def get_skyglow_header():
	headers = (
		"Community Standard Skyglow Data Format 1.0",
		("URL", "http://www.darksky.org/NSBM/sdf1.0.pdf"),
		("Number of header lines", "35"),
		("This data is released under the following license",
			"ODbL 1.0 http://opendatacommons.org/licenses/odbl/summary/"),
		("Device type", CONFIG.get("devicetype")),
		("Instrument ID", CONFIG.get("stationid")),
		("Data supplier", CONFIG.get("operatorname")),
		("Location name", CONFIG.get("locationname")),
		("Position (lat, lon, elev(m))", "%s, %s, %s"%(
			CONFIG.get("latitude"), CONFIG.get("longitude"),
			CONFIG.get("elevation"))),
		("Local timezone", CONFIG.get("timezone")),
		("Time Synchronization", CONFIG.get("syncmethod")),
		("Moving / Stationary position", "STATIONARY"),
		("Moving / Fixed look direction", "FIXED"),
		("Number of channels", "1"),
		("Filter per channel", "None"),
		("Measurement direction per channel", CONFIG.get("pointing")),
		("Field of view (degrees)", "180"),
		("Number of fields per line", "4"),
		("IYAL serial number", "n/a"),
		("IYAL firmware version", "n/a"),
		("IYAL readout test", "n/a"),
		("Comment", ""),  # Do we want to support those?
		("Comment", ""),
		("Comment", ""),
		("Comment", ""),
		("Comment", ""),
		"blank line 27",
		"blank line 28",
		"blank line 29",
		"blank line 30",
		"blank line 31",
		"blank line 32",
		"UTC Date & Time, Local Date & Time, Temperature, Counts",
		"YYYY-MM-DDTHH:mm:ss.fff;YYYY-MM-DDTHH:mm:ss.fff;Celsius;counts",
		"END OF HEADER")

	lines = []
	for header in headers:
		if not isinstance(header, str):
			header = "%s: %s"%header
		lines.append("# %s\r\n"%header)

	return "".join(lines)


def make_record_factory(lightmeter):
	"""returns function producing skyglow records from a lightmeter.

	You'll need to prepend the result of get_skyglow_header to whatever
	you get from here.

	Note that the record returned by the function is complete, i.e.,
	it already has a trailing lf.
	"""
# FIXME: actually make this skyglow compatible
	def make_record():
		return "%s;%s;%.1f;%d\r\n"%(
			time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()),
			time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime()),
			lightmeter.get_temperature(),
			lightmeter.get_light())
	return make_record


class FileManager(object):
	"""A class that manages the data files.

	These are constructed with a directory that must be writable by the current
	user.  Within it, there is a directory  cur, into which the logger
	writes, a directory queue containing files waiting for the upload,
	and a directory archive containing the files that have been uploaded.

	root_dir may contain ~[user] (it is put through os.path.expanduser).
	"""
	def __init__(self, root_dir, local=True):
		self.root_dir = os.path.expanduser(root_dir)
		self.local = local

		self.cur_dir = os.path.join(self.root_dir, "cur")
		self.queue_dir = os.path.join(self.root_dir, "queue")
		self.archive_dir = os.path.join(self.root_dir, "archive")
		self.upload_report = os.path.join(self.root_dir, "upload.report")
		self.current_log = None
		self.queue_lock = threading.Lock()
		self.queue_children = set()
		self.do_flush = False
		self._make_dirs()
		self._install_signal_handler()

	def _install_signal_handler(self):
		# arranges for a flush of the current data file to be scheduled
		# when SIGUSR1 is sent.
		def schedule_flush(signo, stack):
			self.do_flush = True
		signal.signal(signal.SIGUSR1, schedule_flush)

	def _make_dirs(self):
		# A helper for __init__
		for att in ["cur_dir", "queue_dir", "archive_dir"]:
			path = getattr(self, att)
			if not os.path.isdir(path):
				os.makedirs(path)

	def _get_logger_name(self):
		return os.path.join(self.cur_dir, "curdata.lm")

	def get_logger_file(self):
		"""returns a file instance suitable to write the current data to.

		This is the same file if this is called several times; however
		after a rotation a different file will be returned.

		It's probably a good idea to fm.get_logger_file().write always
		(rather than keep the reference to what's returned here).
		"""
		if self.current_log is None:
			self.current_log = open(self._get_logger_name(), "w", encoding="utf-8")
			self.current_log.write(get_skyglow_header())
		if self.do_flush:
			self.current_log.flush()
			self.do_flush = False
		return self.current_log

	def rotate(self):
		"""rotates away the data data currently logged.

		Well behaved clients will be logging to a new file after this
		functions has finished.  This function should be thread-safe.
		"""
		# if there's no current file, don't bother
		if not os.path.exists(self._get_logger_name()):
			return

		# rename the current file so it's out of the way; clients still
		# log to it
		fd, tmp_name = tempfile.mkstemp(dir=self.cur_dir)
		os.close(fd)
		os.rename(self._get_logger_name(), tmp_name)

		# with this, the current log will be reopened next time it is
		# requested
		self.current_log = None

		# now move the old file to the upload queue
		os.chmod(tmp_name, 0o664)
		with self.queue_lock:
			while True:
				dest_name = os.path.join(self.queue_dir,
					datetime.datetime.utcnow().strftime(
						"%%Y%%m%%d_%%H%%M%%S_%s.skyglow"%CONFIG.get("stationid")))
				if os.path.exists(dest_name):
					time.sleep(1)
				else:
					break

			os.rename(tmp_name, dest_name)

	def _collect_dead_children(self):
		"""collects finished uploader children

		This is called on each upload and should ensure there's not more
		than one child process hanging around waiting for us to acknowlede them
		(unless we spawn them faster than they can die...).
		"""
		reaped = set()
		for pid in list(self.queue_children):
			try:
				res = os.waitpid(pid, os.WNOHANG)
				if res!=(0, 0):
					reaped.add(pid)
			except os.error:
				# let's assume this broke in a way that relieves us from
				# responsibility
				reaped.add(pid)
		self.queue_children = self.queue_children-reaped

	def _guess_error(self, server_reply):
		"""guesses the error message from the server.

		I don't want to depend on BeautifulSoup, and the server side
		doesn't have a proper machine-readable interface (yet).
		Let's to re-based screen scraping like it's 1999.
		"""
		mat = re.search('<div class="errors">(.*?)</div>', server_reply)
		if mat:
			mat = re.search("<li>(.*?)</li>", mat.group(1))
			if mat:
				return mat.group(1)
		return "Error in content or transfer"

	def _raise_if_upload_error(self, upload_size, upload_name, server_response):
		"""raises an UploadError if server_response does not look like the
		upload was successful.
		"""
		if server_response.status_code!=200:
			raise UploadError("Bad status %s (404 in all likelihood means bad"
				" credentials)"%server_response.status)
		expectation = "File %s uploaded, %d bytes"%(
			upload_name, upload_size)

		if expectation not in server_response.text:
			raise UploadError(self._guess_error(server_response.text))

	def _upload_one(self, path):
		"""tries to upload and de-queue path.
		"""
		LOGGER.info("Uploading '%s'"%path)
		content, file_name = preprocess_file(path)

		response = requests.post(CONFIG.get_upload_url(),
			{"_charset_": "UTF-8", "__nevow_form__": "upload"},
			files=[("inFile", (file_name, io.BytesIO(content)))])

		self._raise_if_upload_error(
			len(content), file_name, response)
		os.rename(path,
			os.path.join(self.archive_dir, os.path.basename(path)))

	def run_upload_in_process(self):
		"""uploads stuff in the queue.

		Don't run this from within the data collection, it will block.

		There must not be two uploaders running on the same directory.
		We don't enforce this right now, but we probably should (just have
		a lockfile?)
		"""
		if self.local:
			return

		to_upload = glob.glob(os.path.join(self.queue_dir, "*.skyglow"))
		while to_upload:
			if not to_upload:
				break
			# Try each only once
			curFile = to_upload.pop()
			with open(self.upload_report, "a") as f:
				f.write("Uploading %s..."%curFile)
				try:
					self._upload_one(curFile)
					f.write(" ok\n")
				except UploadError as ex:
					traceback.print_exc(file=f)
					f.write("upload error -- \n"+str(ex)+"\n\n")
				except Exception as ex:
					traceback.print_exc(file=f)
					f.write("internal failure --\n"+str(ex)+"\n\n")
			time.sleep(10)

	def upload(self):
		"""spawns a process that attempts to upload data that may have
		accumulated in the queue.
		"""
		if self.local:
			return

		LOGGER.info("Uploading from queue")
		try:
			os.unlink(self.upload_report)
		except os.error:
			pass

		pid = os.fork()
		if pid==0:
			try:
				self.run_upload_in_process()
			finally:
				os._exit(0)
		else:
			# parent: manage children
			self.queue_children.add(pid)
			self._collect_dead_children()

	def get_info_dict(self):
		"""returns a dictionary of file manager parameters.

		This is used in the fillers for the templates.
		"""
		fsstats = os.statvfs(self.cur_dir)

		return {
			"last_upload":
				time.strftime(TIME_FORMAT,
					time.localtime(os.path.getmtime(self.archive_dir))) or "N/A",
			"no_queued": len(os.listdir(self.queue_dir)),
			"no_archived": len(os.listdir(self.archive_dir)),
			"log_drive_space": "%.2fM"%(fsstats.f_bavail*fsstats.f_bsize/1e6),
		}

	def _get_with_dates(self, dir):
		"""returns pairs of (timestamp, name) for the files in dir.
		"""
		return [(os.path.getmtime(name), name)
			for name in (
				os.path.join(dir, name) for name in os.listdir(dir))]

	def get_newest_files(self, max_size):
		"""returns the paths of the most recent data files.

		The function stops collecting after more than max_size bytes
		are in the files.

		This never includes the current log, as that may include incomplete
		lines and all kinds of other unpleasantness.

		This races with uploads.  This means files may be vanishing, and
		there could even be os.errors coming out of this.  Hence, as long
		as there's the possibility of uploads running, this is not dependable.

		The files are oldest-files-first.
		"""
		candidates = self._get_with_dates(self.queue_dir
			)+self._get_with_dates(self.archive_dir)
		candidates.sort()

		result, cur_size = [], 0
		while cur_size<max_size and candidates:
			_, name = candidates.pop()
			if os.path.exists(name):
				result.append(name)
				cur_size += os.path.getsize(name)
		result.reverse()
		return result



################### upload helpers

def compress(data, f_name):
	"""returns the content of f_name as a gzip-readable string.
	"""
	compr = io.BytesIO()
	g = gzip.GzipFile(f_name, "w", 5, compr)
	g.write(data)
	g.close()
	return compr.getvalue()


def preprocess_file(file_name):
	"""normalizes the content of file_name and returns it and a possibly
	mogrified name.

	Basically, we remove all CRs and compress the result to something gzip can
	handle if it's not compressed yet.
	"""
	with open(file_name, "rb") as f:
		content = f.read()
	if not file_name.endswith(".gz"):
		content = compress(content.replace(b"\r", b""), file_name)
		file_name = file_name+".gz"
	return content, file_name


def collect_until(stop_time, record_factory, file_manager, interval):
	"""writes stuff read from record_factory every interval seconds until
	stop_time UTC.
	"""
	while time.time()<stop_time:
		file_manager.get_logger_file().write(
			record_factory())
		time.sleep(interval)


def main_loop(get_skyglow_record, file_manager):
	"""the endless loop fetching and writing data.
	"""
	while True:
		collect_until(
			time.time()+CONFIG.get("uploadinterval")*24*3600,
			get_skyglow_record, file_manager,
			CONFIG.get("sampleinterval"))
		file_manager.rotate()
		file_manager.upload()


################## Web interface


def escapePCDATA(val):
	if val is None:
		return ""

	return str(val
		).replace("&", "&amp;"
		).replace('<', '&lt;'
		).replace('>', '&gt;'
		).replace("\0", "&x00;")


def escapeAttrVal(val):
	"""returns val with escapes for double-quoted attribute values.
	"""
	if val is None:
		return ""

	return escapePCDATA(val).replace('"', '&quot;')


class Template(object):
	"""a *very* basic and ad-hoc template engine.

	It works on HTML files, with the following constructs expanded:

	* $[key] -- value for key, escaped for double-quoted att values
	* $(key) -- value for key, escaped for PCDATA
	* $|func| -- replace with the value of func(vars)
	* $?key? -- checked="checked" if vars[key], empty otherwise
	* $!raw! -- value for key, non-escaped (other template ops are expanded)
	* $^ -- standard header declarations (use style.css)
	* $$ -- a $ char.
	"""
	def __init__(self, source):
		self.source = source

	@classmethod
	def fromFile(cls, srcFile):
		with open(srcFile, encoding="utf-8") as f:
			return cls(f.read())

	def render(self, vars):
		"""returns a string with the template filled using vars.

		vars is a dictionary mapping keys to unicode-able objects.
		"""
		return re.sub(r"\$\|([a-zA-Z0-9_]+)\|",
				lambda mat: globals()[mat.group(1)](vars),
			re.sub(r"\$\(([a-zA-Z0-9_]+)\)",
				lambda mat: escapePCDATA(vars.get(mat.group(1), "")),
			re.sub(r"\$\[([a-zA-Z0-9_]+)\]",
				lambda mat: escapeAttrVal(vars.get(mat.group(1), "")),
			re.sub(r"\$\?([a-zA-Z0-9_]+)\?",
				lambda mat: (vars.get(mat.group(1)) and 'checked="checked"') or "",
			re.sub(r"\$!([a-zA-Z0-9_]+)!",
				lambda mat: str(vars.get(mat.group(1), "")),
			self.source))))).replace(
			"$^", STANDARD_HEADER_MATERIAL).replace(
			"$$", "$")


class HTTPHandler(http.server.BaseHTTPRequestHandler):
	"""The handler for requests to the built-in web server.
	"""
#	protocol_version = "HTTP/1.1" (see where content size computation fails)

	def get_info_dict(self, title):
		"""returns a dictionary containing sundry items of interest on the
		lightmeter and the readout client.

		The value returned here is passed to most templates.

		TODO: document keys available.
		"""
		res = CONFIG.settings.copy()
		del res["accesskey"]
		res.update(self.server.file_manager.get_info_dict())
		res.update(self.server.lightmeter.get_info_dict())
		res.update({
			"title": title,
			"utcnow": datetime.datetime.utcnow().strftime(TIME_FORMAT),
			"localtime": datetime.datetime.now().strftime(TIME_FORMAT),
		})
		return res

	def _serveContent(self, responseCode, contentType,
			content, encoding=None, additionalHeaders={}):
		"""serves any content in a unicode string in UTF8.
		"""
		if encoding:
			content = content.encode(encoding)

		if isinstance(content, str):
			content = content.encode("utf-8")

		self.send_response(responseCode)
		self.send_header("Content-type", contentType)
		self.send_header("Content-length", "%d"%len(content))
		for key, value in additionalHeaders.items():
			self.send_header(key, value)
		self.end_headers()
		self.wfile.write(content)

	def _serveError(self, errCode, msg):
		content = Template(ERROR_TEMPLATE).render({
				"title": "Error",
				"msg": msg,
			})
		self._serveContent(errCode, "text/html; charset=utf-8",
			content, "utf-8")

	def _serveRedirect(self, target):
		self._serveContent(301, "text/html; charset=utf-8",
			"", "utf-8", {"Location": target})

	def do_config(self, args=None):
		"""serves a configuration dialog and saves changed values if args
		(a dict as from a filled-out form) is given.
		"""
		self._serveContent(401, "text/plain", "Please Authenticate",
			{"WWW-Authenticate": 'Basic realm="mlum"'})
		return
#		self._serveContent(200, "text/html; charset=utf-8",
#			Template(CONFIG_TEMPLATE).render(self.get_info_dict("Lightmeter info")))

	def get_current_plot(self):
		"""returns PNG data for the data recorded in the last 24 hours
		(or since restart).

		This is cached for 10 minutes, as it's fairly CPU intensive.
		"""
		if self.server.last_plot:
			genTime, value = self.server.last_plot
			if time.time()-genTime<10*60:
				return value

		cur = datetime.datetime.now()
		curHour = cur.hour+cur.minute/60.+cur.second/3600.
		genTime = time.time()
		times, fluxes = [], []
		for ts, flux in self.server.lightmeter.recorder.buffer:
			try:
				times.append(curHour+(ts-genTime)/3600.)
				fluxes.append(flux)
			except:
				sys.stderr.write("Bad input: %s %s\n"%(repr(ts), repr(flux)))

		fig = pyplot.figure()
		ax = fig.add_subplot(111)
		ax.semilogy(times, fluxes)
		ax.set_title("Recorded measurements from %s"%CONFIG.get("stationid"))
		ax.set_xlabel("local time [hours]")
		if CONFIG.get("calibrated"):
			ax.set_ylabel("Flux [W/m^2]")
		else:
			ax.set_ylabel("Flux [arbitrary units]")

		rendered = io.BytesIO()
		pyplot.savefig(rendered, format="png", dpi=40)
		pyplot.close()
		img = rendered.getvalue()
		self.server.last_plot = (genTime, img)
		return img

	def real_do_GET(self):
		"""handles GET requests.

		This method is primarily supposed to handle all kinds of
		errors and do something sensible with them.  The real work
		is done by _serveRequest.
		"""
		if self.path=="/style.css":
			return self._serveContent(200, "text/css", STYLE)

		elif self.path=="/":
			self._serveContent(200, "text/html; charset=utf-8",
				Template(ROOT_TEMPLATE).render(self.get_info_dict("Lightmeter info")))

		elif self.path=="/upload.report":
			content = "No report at this time, sorry"
			try:
				with open(self.server.file_manager.upload_report) as f:
					content = f.read()
			except IOError:
				pass
			self._serveContent(200, "text/plain; charset=utf-8", content)

		elif self.path=="/current.png":
			self._serveContent(200, "image/png",
				self.get_current_plot())

		elif self.path=="/info.json":
			import json  # import here so we don't fail totally on python 2.5
			self._serveContent(200, "text/json",
				json.dumps(self.get_info_dict("Lightmeter info")))

# this doesn't work yet (need to define CONFIG_TEMPLATE, work out auth
#		elif self.path=="/config":
#			self.do_config()

		else:
			self._serveError(404, msg="The path you specified is unknown")

	def do_GET(self):
		try:
			self.real_do_GET()
		except Exception as ex:
			traceback.print_exc()
			self._serveError(500, str(ex))


def startHTTPServer(port, file_manager, lightmeter):
	"""spawns off a (daemon) thread serving info pages via HTTP.
	"""
	httpd = http.server.HTTPServer(("", port), HTTPHandler)
	httpd.file_manager, httpd.lightmeter = file_manager, lightmeter
	httpd.last_plot = None
	serverThread = threading.Thread(target=httpd.serve_forever)
	serverThread.setDaemon(True)
	serverThread.start()

	def close():
		httpd.server_close()
		serverThread.join(2)

	return close


################### Daemonizing

def _drop_privileges():
	"""changes to whatever user is configured if we're running as root.

	If no user is configured, bail out (don't run as root, it's too
	tricky). If not running as root, don't do anything.

	This doesn't work very well right now as we're looking for our
	configuration file and stuff in the call user's home, which would be
	root in the case of privilege dropping.  So, don't use it, always
	start the daemon as a user.
	"""
	if os.getuid(): # not root, do nothing
		return

	destUser = CONFIG.get("user")
	if not destUser:
		LOGGER.error("Server started as root but no 'user'"
			" defined.  Refusing to run as root, bailing out.")
		sys.exit(1)

	try:
		uid = pwd.getpwnam(destUser)[2]
	except KeyError:
		LOGGER.error("Server user %s does not exist on this"
			" system.  Refusing to run as root, bailing out."%destUser)
		sys.exit(1)
	os.setuid(uid)


class _PIDManager(object):
	"""A manager for the PID of the server.

	There's a single instance of this below.
	"""
	def __init__(self):
		self.path = os.path.expanduser("~/.lightmeter.pid")

	def get_pid(self):
		"""returns the PID of the currently running server, or None.
		"""
		try:
			with open(self.path) as f:
				pid_string = f.readline()
		except IOError: # PID file does not exist (or we're beyond repair)
			return None
		try:
			return int(pid_string)
		except ValueError: # junk in PID file -- no sense in keeping it
			LOGGER.warning("%s contained garbage, attempting to unlink"%
				self.path)
			self.clear_pid()

	def set_pid(self):
		"""writes the current process' PID to the PID file.

		Any existing content will be clobbered; thus, you could have
		races here.  Fix this at some point (of course, HOME could be
		on NFS...)
		"""
		try:
			with open(self.path, "w") as f:
				f.write(str(os.getpid()))
		except IOError: # Cannot write PID.  Since the pid file at on HOME,
				# this suggests we'd not do anything useful anyway.
			logging.error("Cannot write PID file %s."
				" This is bad, bailing out."%self.path)
			sys.exit(1)

	def clear_pid(self):
		"""removes the PID file.
		"""
		try:
			os.unlink(self.path)
		except os.error as ex:
			if ex.errno==2: # ENOENT, we don't have to do anything
				pass
			else:
				logging.error("Cannot remove PID file %s (%s)."%(
						self.file, str(ex)))


PID_MANAGER = _PIDManager()

def _wait_for_process_exit(timeout=5):
	"""waits for the server process to remove its pid file.

	If the process doesn't do this within timeout seconds, the function
	terminates with an error message.
	"""
	for i in range(int(timeout*10)):
		if PID_MANAGER.get_pid() is None:
			break
		time.sleep(0.1)
	else:
		sys.exit("The server with pid %d appears not to die.  Please check"
			" manually and remove %s when it's dead."%(
				PID_MANAGER.get_pid(), PID_MANAGER.path))


def _stop_server():
	"""helps cmd_stop.
	"""
	pid = PID_MANAGER.get_pid()
	if pid is None:
		sys.exit("Lightmeter server doesn't seem to run.")

	try:
		os.kill(pid, signal.SIGTERM)
	except os.error as ex:
		if ex.errno==3: # no such process
			pid = PID_MANAGER.clear_pid()
			warnings.warn("Unclean server shutdown.  Removed serverPID.")
			return
		else:
			raise
	_wait_for_process_exit()
	sys.exit(0)


def cmd_stop():
	"""attempts to stop a running lightmeter.
	"""
	try:
		_stop_server()
	except AttributeError:
		sys.exit("Server doesn't seem to run, nothing killed.")
	except Exception as ex:
		sys.exit("Could not kill server (%s)."%ex)
	sys.exit(0)


def daemonize(callable):
	"""Does the double detach and runs callable.

	This function will exit the process if the presence of a PID file indicates
	another server is already running.
	"""
	if PID_MANAGER.get_pid() is not None:
		# PID file -- let's see if a process with this PID is at least running
		try:
			os.getsid(PID_MANAGER.get_pid())
		except os.error:
			# process doesn't exist, just remove the pid file
			PID_MANAGER.clear_pid()
		else:
			sys.exit("It seems there's already a server (pid %s) running."
				" Try %s stop."%(
					PID_MANAGER.get_pid(),
					sys.argv[0]))

	# We translate TERMs to INTs to ensure finally: code is executed
	signal.signal(signal.SIGTERM,
		lambda a,b: os.kill(os.getpid(), signal.SIGINT))
	pid = os.fork()
	if pid == 0:
		os.setsid()
		pid = os.fork()
		if pid==0:
			PID_MANAGER.set_pid()
			atexit.register(PID_MANAGER.clear_pid)

			os.close(0)
			os.close(1)
			handle = os.open("/dev/null", os.O_RDWR)
			os.dup(handle)
			os.dup(handle)
			os.dup(handle)
			# os.dup some stderr file for the server here?
			callable()
		else:
			os._exit(0)
	else:
		os._exit(0)


################### User interface

def parse_command_line():
	from optparse import OptionParser
	usage = '%prog [options] start|stop|foreground|initscript|debug'
	parser = OptionParser(usage=usage, version=VERSION)

	parser.add_option('-v', '--VERBOSE', action='store_true',
		dest='verbose', help='Talk while working', default=False)
	parser.add_option('-u', '--upload-only', action='store_true',
		dest='upload_only', help='Only upload what\'s in the queue, then exit.',
		default=False)
	parser.add_option('-l', '--local', action='store_true',
		dest='local', help='Do no uploads.',
		default=False)
	parser.add_option("", "--fake", action="store_true", dest="fake_device",
		help="use blind data generator rather than a real lightmeter")

	opts, args = parser.parse_args()

	if len(args)==1 and args[0]=="initscript":
		print(INITSCRIPT)
		sys.exit(0)

	if len(args)!=1 or args[0] not in ["start", "stop", "foreground", "debug"]:
		parser.print_help()
		sys.exit(1)

	return opts, args


def run_debug():
	"""just reads the lightmeter and prints results.
	"""
	l = Lightmeter([])
	while True:
		print(l.get_light())
		print(l.get_temperature())
		time.sleep(1)


def main_raising(opts, file_manager):
	"""the program without a toplevel exception handler.
	"""
	history_seed = get_last_data(file_manager)

	if opts.fake_device:
		LOGGER.warning("Lightmeter driver uses fake data.  Remove whatever"
			" data this generates before uploading.")
		lightmeter = FakeLightmeter(history_seed)
	else:
		lightmeter = Lightmeter(history_seed)
	get_skyglow_record = make_record_factory(lightmeter)

	file_manager.rotate()

	def run():
		try:
			quit_http = startHTTPServer(
				CONFIG.get("httpport"), file_manager, lightmeter)
			LOGGER.info("Lightmeter driver up and running.  Check"
				" http://localhost:%s"%CONFIG.get("httpport"))
		except KeyboardInterrupt:
			sys.exit(0)
		except:
			LOGGER.error("Lightmeter driver failed to start HTTP server"
				" (maybe something is already running on the port?)")

		try:
			main_loop(get_skyglow_record, file_manager)
		finally:
			quit_http()

	if opts.foreground:
		run()
	else:
		daemonize(run)


def main():
	opts, args = parse_command_line()
	if opts.verbose:
		CONFIG.verbose = True
	CONFIG.load()

	file_manager = FileManager(CONFIG.get("rootdir"),
		local=opts.local)
	if opts.upload_only:
		CONFIG.verbose = True
		file_manager.run_upload_in_process()
		sys.exit(0)

	if args[0]=="stop":
		return cmd_stop()
	elif args[0]=="start":
		opts.foreground = False
	elif args[0]=="foreground":
		LOGGER.addHandler(logging.StreamHandler(sys.stderr))
		opts.foreground = True
	elif args[0]=="debug":
		run_debug()
		return
	else:
		assert False  # shouldn't have left parse_command_line

	_drop_privileges()

	while True:
		try:
			main_raising(opts, file_manager)
		except KeyboardInterrupt:
			sys.exit(0)

		except libusb1.USBError as msg:
			LOGGER.warning("USB interface error: %s.\n"%msg)
			LOGGER.warning("Re-trying in a minute\n")
			time.sleep(60)

		except Error as msg:
			LOGGER.error("%s\n"%msg)
			sys.exit(1)

		except Exception as msg:
			import traceback; traceback.print_exc()
			LOGGER.error("Severe Error.  Giving up: %s\n"%msg)
			sys.exit(1)


if __name__=="__main__":
	main()

# vim:noet:sw=2:nosta:ts=2