"""
A custom page giving an upload-and-dexter-service.
"""

import glob
import os
import random
import re
import shutil
import subprocess
import urllib.parse

from twisted.internet import defer
from twisted.internet import reactor
from twisted.internet import threads
from twisted.web import server
from twisted.web import static
from twisted.web import template
from twisted.web.template import tags as T

from gavo import base
from gavo import formal
from gavo import svcs
from gavo import utils
from gavo import web

NT = formal.addNevowAttributes

stagingPath = os.path.join(base.getConfig("inputsDir"), "dexter", "data")
multipageFormats = set(["pdf"])


class Error(base.Error):
	pass


class ChildFailed(Error):
	pass


class DexterMixin(object):

	sourceName = "unset"

	def _ensureDir(self, dirName):
		if not os.path.exists(dirName):
			os.mkdir(dirName)
		return dirName

	def getResdirName(self):
		return self._ensureDir(
			os.path.join(stagingPath, urllib.parse.quote(self.sourceName)))

	def getPreviewDir(self):
		return self._ensureDir(
			os.path.join(self.getResdirName(), "previews"))

	def getTiffDir(self):
		return self._ensureDir(
			os.path.join(self.getResdirName(), "tiffs"))

	def getResultsDir(self):
		return self._ensureDir(
			os.path.join(self.getResdirName(), "results"))

	def getSourceName(self):
		return os.path.join(self.getResdirName(), "source.bin")

	def getFormatFileName(self):
		return os.path.join(self.getResdirName(), "format.txt")

	def identifySourceFormat(self):
		# I'm not using any mime information since it's usually crap.
		# I might use file, but right now it's just not worth it.
		try:
			with open(self.getSourceName(), "rb") as f:
				magic = f.read(20)
			if magic[:3]==b'\xff\xd8\xff':
				return "jpeg"
			elif magic[:4]==b"\x89PNG":
				return "png"
			elif magic[:3]==b"GIF":
				return "gif"
			elif magic[:4]==b"%PDF":
				return "pdf"
		except IOError: # File somehow damaged, fall through to error
			pass
		raise formal.FieldError("Cannot recognize input format",
			"inFile")

	def getSourceFormat(self, rescan=False):
		if rescan or not os.path.exists(self.getFormatFileName()):
			format = self.identifySourceFormat()
			f = open(self.getFormatFileName(), "w")
			f.write(format)
			f.close()
		return open(self.getFormatFileName()).read()


class ImagePiece(formal.TemplatedPage, DexterMixin):
	def __init__(self, sourceName, page):
		self.sourceName, self.page = sourceName, int(page)
	
	def render(self, request):
		args = request.args
		input = os.path.join(self.getTiffDir(), "%d.tiff"%self.page)
		scale = int(utils.debytify(args.get(b"scale", [1]))[0])
		cmd = ["/usr/local/bin/t2gif", "-bits", "4", "-scale", str(scale)]
		coord = utils.debytify(args.get(b"coord", [""])[0])
		if coord:
			cmd.extend(["-coord",
				",".join(str(int(a)) for a in coord.split(","))])
		cmd.append(input)

		gifStr = subprocess.check_output(cmd, text=False)

		request.setHeader("content-type", "image/gif")
		return gifStr


class ResultPage(formal.TemplatedPage, web.GavoRenderMixin, DexterMixin):
	"""is an abstract base for pages dealing with saved results.

	The results are identified by their source and page, plus a cookie
	in dp_send.
	"""
	def __init__(self, sourceName, page):
		self.sourceName, self.page = sourceName, page
	
	def getResultPath(self, request):
		cookie = utils.debytify(request.args.get(b"pipename", ["0"])[0])
		return os.path.join(self.getResultsDir(), "res%s-%s.txt"%(
			self.page, cookie))


class ResultReceiver(ResultPage):
	"""is a page that receives a result and stores it in a good location.
	"""
	def render(self, request):
		resultPath = self.getResultPath(request)
		tmpName = resultPath+".partial"
		header, postPayload = request.content.getvalue().split(b"\n\n", 1)
		with open(tmpName, "wb") as f:
			f.write(postPayload)
# The whole "permanent/temp" thing is currently a nightmarish hack.
# We'll want to change this in Dexter, but let's wait what ADS says.
		if b"octet-stream" in header:
			os.rename(tmpName, resultPath)
		else:
			os.rename(tmpName, resultPath+".temp")
		request.setHeader("content-type", "text/plain")
		return utils.bytify("Result %s saved.\n"%os.path.basename(resultPath))


class SavedResult(ResultPage):
	"""is a Page delivering previously saved results.
	"""
	retryCount = 0
	contentType = "text/plain"

	def render(self, request):
		self.getResult(request
			).addCallback(self.deliver, request
			).addErrback(self.sendTimeout, request)
		return server.NOT_DONE_YET

	def deliver(self, data, request):
		request.setHeader("content-type", self.contentType)
		request.write(data)
		request.finish()

	def sendTimeout(self, failure, request):
		failure.printTraceback()
		request.setResponseCode(500)
		request.setHeader("content-type", "text/plain")
		request.write("Timeout while waiting for data")
		request.finish()

	def getResult(self, request):
		path = self.getResultPath(request)
		if os.path.exists(path):
			self.contentType = "application/octet-stream"
			with open(path, "rb") as f:
				data = f.read()
			return defer.succeed(data)

		elif os.path.exists(path+".temp"):
			with open(path+".temp", "rb") as f:
				data = f.read()
			os.unlink(path+".temp")
			return defer.succeed(data)

		else:
			self.retryCount += 1
			if self.retryCount>5:
				raise Error("Timeout while waiting for %s"%path)
			d = defer.Deferred().addCallback(lambda res: self.getResult(request))
			reactor.callLater(2, d.callback, request)
			return d


class DexterEdit(formal.TemplatedPage, web.GavoRenderMixin, DexterMixin):
	def __init__(self, sourceName, page):
		self.sourceName, self.page = sourceName, int(page)

	def _getPathValue(self, request, subPath):
		parts = request.uri.decode("utf-8").split("/")
		if not parts[-1]:
			parts.pop()
		parts.pop()
		parts.pop()
		parts.extend((subPath, str(self.page)))
		return base.makeAbsoluteURL("/".join(parts))

	def data_dexterpars(self, request, tag):
		try:
			singlePage = self.getSourceFormat() not in multipageFormats
		except formal.FieldError: # input file damaged
			raise svcs.UnknownURI("No input file for Dexter given")
		if singlePage:
			exScale, width, height = 3, "100", "100"
		else:
			exScale, width, height = 3, "500", "500"
		return {
			"singlePage": str(singlePage),
			"sourceimage": str(self._getPathValue(request, "img"
				)+"?ignored=%s"%str(random.randint(1, 65536))),
					# "ignored" is a mindless hack to work around java's URL
					# class limitations and the cachability of GET requests.
			"receiverURL": self._getPathValue(request, "receive"),
			"senderURL": self._getPathValue(request, "send"),
			"helpURL": "/dexter/ui/ui/static/Dexterhelp.html",
			"sourceName": self.sourceName,
			"exScale": exScale,
			"page": str(self.page+1),
			"width": width,
			"height": height,
			"codebase": "/dexter/ui/ui/static",
			"archive": "Dexter.jar",
		}

	loader = template.TagLoader(T.html[
		T.head[
			T.title["DfyD: Extraction"],
			T.transparent(render="commonhead"),
			T.script(src="https://cjrtnc.leaningtech.com/2.1/loader.js"),
			T.script["""
				window.addEventListener("load", function() {
					cheerpjInit();
					var appletEl = document.querySelectorAll('.cheerpjApplet')[0];
					appletEl.height = window.innerHeight-appletEl.offsetTop+"px";
				}, false);
				"""],
			T.style(type="text/css")["""
				.cheerpjApplet {
					width:66%;
				}
				""",]
		],
		NT(T.body, data="dexterpars")(render="mapping")[
			T.div(style="position:fixed;top:0pt;right:0pt;background:white")[
				T.a(href="/dexter/ui/ui/info")["Service Info"]],
			T.h1["Dexter: Data Extraction"],
			T.p["Please have at least a glance at ",
				T.a(href="/dexter/ui/ui/static/Dexterhelp.html")["Help for Dexter"],
				"."],
			template.Tag("cheerpj-applet")(codebase=template.slot("codebase"),
					archive=template.slot("archive"),
					code="Dexter",
					width=template.slot("width"),
					height=template.slot("height"),
					style="float:left")[
				T.param(name="SOURCEIMAGE", value=template.slot("sourceimage")),
				T.param(name="RECEIVERURL", value=template.slot("receiverURL")),
				T.param(name="SENDERURL", value=template.slot("senderURL")),
				T.param(name="HELPURL", value=template.slot("helpURL")),
				T.param(name="DEFAULTSCALE", value="8"),
				T.param(name="EXDEFAULTSCALE", value=template.slot("exScale")),
				T.param(name="BIBCODE", value=template.slot("sourceName")),
				T.param(name="PAGE", value=template.slot("page")),
				T.param(name="SKIPSELECTION", value=template.slot("singlePage")),
			]
		],
		T.div(style="float:right;width: 30%;overflow:scroll")[
			T.p[T.a(href="..")["Back to selection"]],
			T.h2["Extracted Data"],
			T.p["This will be filled in when you press the 'Send' button in"
				" Dexter."],
			T.iframe(name="dp_save"),
		]
	])


def raisingSystem(command):
	child = subprocess.Popen(command, shell=True, stderr=subprocess.STDOUT,
		stdout=subprocess.PIPE)
	response, _ = child.communicate()
	if child.returncode:
		raise ChildFailed("Shell command failed, message: %s"%
			utils.makeEllipsis(repr(response), 60))


# Allowed dithering strategies, first is default, see binMeth field below
ditherers = [("threshold", "Thresholding"), ("fs", "Floyd-Steinberg"),
	("dither8", "Ordered Dither"), ("hilbert", "Hilbert halftoning"),]
_ditherDict = dict(ditherers)


def prepareUsingPNM(toPNMProgram, pg, parameters):
	binMeth = parameters.get("binMeth")
	if binMeth not in _ditherDict:
		binMeth = "threshold"
	thresholder = "pgmtopbm -%s"%binMeth
	scale = min(20, max(0.1, 600/float(parameters.get("assumeDPI", 600))))
	if abs(scale-1)<0.1:
		scaler = ""
	else:
		scaler = "pnmscale %f | "%scale
	raisingSystem("%s %s | ppmtopgm | %s %s |"
		" pnmtotiff -g4 -rowsperstrip 50000 > %s"%(toPNMProgram,
			pg.getSourceName(), scaler, thresholder,
			os.path.join(pg.getTiffDir(), "0.tiff")))
	raisingSystem("tifftopnm %s | pnmscale -xsize 150 |  pnmtopng > %s"%(
		os.path.join(pg.getTiffDir(), "0.tiff"),
		os.path.join(pg.getPreviewDir(), "0.png")))


def prepareJpeg(pg, parameters):
	prepareUsingPNM("djpeg", pg, parameters)


def prepareGif(pg, parameters):
	prepareUsingPNM("giftopnm", pg, parameters)


def preparePng(pg, parameters):
	prepareUsingPNM("pngtopnm", pg, parameters)


def preparePdf(pg, parameters):
	tiffDir, previewDir = pg.getTiffDir(), pg.getPreviewDir()
	raisingSystem("pdftoppm -l 20 -r 600 -mono -q %s %s"%(
		pg.getSourceName(), os.path.join(tiffDir, os.path.join(tiffDir, "pg"))))
	for ind, fName in enumerate(glob.glob(os.path.join(tiffDir, "pg*.pbm"))):
		raisingSystem("pnmtotiff -g4 -rowsperstrip 50000 %s > %s"%(
			fName, os.path.join(tiffDir, "%d.tiff"%ind)))
		raisingSystem("pnmscale -xsize 100 %s |  pnmtopng > %s"%(
			fName, os.path.join(previewDir, "%d.png"%ind)))
		os.unlink(fName)


# source preparers are functions taking something mixing in DexterMixin
# and a dictionatry containing parameters.  You should assume as little
# as possible about the contents of that dictionary, but realistically,
# it will be the data coming in from the form.
sourcePreparers = {
	"jpeg": prepareJpeg,
	"pdf": preparePdf,
	"png": preparePng,
	"gif": prepareGif,
}


class ConfirmationAsker(formal.ResourceWithForm, web.GavoRenderMixin,
		DexterMixin):
	"""is a page asking for confirmation of an action.

	This abstract, inheriting classes have to fill in:
	* render_title
	* render_head
	* acceptLabel
	* confirmLabel
	* a doAction method

	Things have to be arranged such that going to the parent of the
	given page will take you back to the page in question (or override goBack).

	You need to return a sensible redirect from your doAction method.
	"""
	def __init__(self, sourceName):
		self.sourceName = sourceName
		super(ConfirmationAsker, self).__init__()

	def crash(self, failure, request):
		return web.renderDCErrorPage(failure, request)

	def goBack(self, request, form, data):
		raise svcs.SeeOther(utils.debytify(
			b"/".join(request.path.split(b"/")[:-1])))

	def form_confirmation(self, request):
		form = formal.Form()
		form.addAction(self.doAction, name="goAhead", label=self.acceptLabel)
		form.addAction(self.goBack, name="cancel", label=self.rejectLabel)
		return form

	loader = template.TagLoader(T.html[
		T.head[
			T.title(render="title"),
			T.transparent(render="commonhead"),
		],
		T.body[
			T.h1(render="head"),
			T.transparent(render="form confirmation"),
		]
	])


class ExtDeleter(ConfirmationAsker):
	"""is a page deleting individual extraction actions.
	"""
	acceptLabel = "Confirm deletion"
	rejectLabel = "Cancel deletion"

	def __init__(self, sourceName, resultName):
		self.resultName = os.path.basename(resultName)
		super(ExtDeleter, self).__init__(sourceName)

	@template.renderer
	def title(self, request, tag):
		return tag["DfyD: Confirm deletion of extracted data"]

	@template.renderer
	def head(self, request, tag):
		return tag["Dexter: Confirm deletion of %s"%self.resultName]

	def doAction(self, request, form, data):
		targetFile = os.path.join(self.getResultsDir(), self.resultName)
		try:
			os.unlink(targetFile)
		except os.error:
			pass
		return self.goBack(request, form, data)


class DataDeleter(ConfirmationAsker):
	acceptLabel = "Confirm deletion"
	rejectLabel = "Cancel deletion"

	@template.renderer
	def title(self, request, tag):
		return tag["DfyD: Confirm deletion of data set"]

	@template.renderer
	def head(self, request, tag):
		return tag["Dexter: Confirm deletion of data set %s"%self.sourceName]

	def doAction(self, request, form, data):
		dir = self.getResdirName()
		shutil.rmtree(dir, ignore_errors=True)
		return self.goBack(request, form, data)


class DocumentPage(formal.ResourceWithForm, web.GavoRenderMixin,
		DexterMixin):
	"""is a renderer for document-global properties.
	"""
	
	passedMime = None

	def __init__(self, sourceName):
		self.sourceName = urllib.parse.unquote(utils.debytify(sourceName))
		formal.ResourceWithForm.__init__(self)

	def crash(self, failure, request):
		return web.renderDCErrorPage(failure, request)

	def data_pages(self, request, tag):
		return list(range(
			len(glob.glob(os.path.join(self.getPreviewDir(), "*.png")))))

	def data_extracted(self, request, tag):
		return [os.path.basename(n) for n in os.listdir(self.getResultsDir())
			if n.endswith(".txt")]

	@template.renderer
	def imagelink(self, request, tag):
		baseuri = request.uri.decode("utf-8")
		dexterLink = baseuri+"/edit/%s"%tag.slotData
		imageLink = baseuri+"/p/"+str(tag.slotData)
		return tag(href=dexterLink)[T.img(src=imageLink)]

	@template.renderer
	def resultlink(self, request, tag):
		title = "Data from page %s"%re.search(r"\d+", tag.slotData).group()
		dataLink = utils.debytify(request.uri)+"/retrieve/"+tag.slotData
		return tag(href=dataLink)[title]

	@template.renderer
	def removelink(self, request, tag):
		removeLink = utils.debytify(request.uri)+"/remove/"+tag.slotData
		return tag(href=removeLink)
	
	@template.renderer
	def childLink(self, request, tag):
		tag.attributes["href"] = (
			request.path
			+b"/"
			+(tag.attributes["href"].encode("ascii")))
		return tag

	def form_upload(self, request):
		# We want this as an attribute since we may want to add errors
		self.uplform = formal.Form()
		self.uplform.addField("inFile", formal.File(required=True),
			label="Source File", description="An image or a PDF file")
		self.uplform.addField("assumeDPI", formal.Integer(missing=600),
			label="Assume DPI", description="For images: assume this resolution"
				" (in dpi; see also service info)")
		self.uplform.addField("binMeth", formal.String(missing="thresh"),
			label="Binarization method", description="For images:"
				" Strategy when converting image to black/white (see service"
				" info)", widgetFactory=formal.widgetFactory(
					formal.SelectChoice, options=ditherers[1:],
					noneOption=ditherers[0]))
		for name, value in [("assumeDPI", 600), ("binMeth", "threshold")]:
			if name not in self.uplform.data:
				self.uplform.data[name] = value
		self.uplform.addAction(self.submitAction, label="Upload")
		return self.uplform

	def _clearInDir(self, dir, pattern):
		for name in glob.glob(os.path.join(dir, pattern)):
			os.unlink(name)

	def clearImgCache(self):
		self._clearInDir(self.getPreviewDir(), "*.png")
		self._clearInDir(self.getTiffDir(), "*.tiff")
		self._clearInDir(self.getTiffDir(), "*.pbm")

	def prepareSource(self, parameters):
		try:
			self.clearImgCache()
			sourcePreparers[self.getSourceFormat()](self, parameters)
		except ChildFailed:
			self.clearImgCache()
			raise formal.FieldError("Image generation on source failed --"
				" probably a corrupted file.", "inFile")
		
	def handleNewSource(self, data):
		"""generates or updates all ephemeral data for the current source.
		"""
		try:
			self.getSourceFormat(rescan=True)
		except formal.FieldError:
			os.unlink(self.getSourceName())
			raise
		except Exception as msg:
			raise formal.FieldError("Could not prepare source (%s) --"
				" you may want to report this"%repr(str(msg)), "inFile")
		self.prepareSource(data)

	def submitAction(self, request, form, data):
		_, srcFile = data["inFile"]
		with open(self.getSourceName(), "wb") as f:
			f.write(srcFile.read())
		return threads.deferToThread(self.handleNewSource, data
			).addCallback(self._reDisplay, request
			).addErrback(self._flagError, request
			).addErrback(web.renderDCErrorPage, request)

	def _reDisplay(self, result, request):
		raise svcs.SeeOther(request.path)
		
	def _flagError(self, failure, request):
		if isinstance(failure.value, base.ExecutiveAction):
			# let weberrors handle redirects and such
			return failure
		if isinstance(failure.value, (formal.FormError, formal.FieldError)):
			self.uplform.errors.add(failure.value)
		else:
			self.uplform.errors.add(formal.FormError(str(failure.value)))
		return self

	def getMyURL(self):
		return f"/dexter/ui/ui/custom/{self.sourceName}"

	def getChild(self, name, request):
		segments = request.popSegments(name)
		if name==b"":
			# must redirect so relative links are ok
			raise svcs.SeeOther(self.getMyURL())
		elif name==b"retrieve":
			resName = os.path.join(self.getResultsDir(), utils.debytify(segments[1]))
			return static.File(resName)
		elif name==b"remove":
			if len(segments)==2:
				return ExtDeleter(self.sourceName, segments[1])
			else:
				raise svcs.SeeOther(self.getMyURL())
		elif name==b"purgeData":
			return DataDeleter(self.sourceName)
		elif name==b'p':
			tp = os.path.join(self.getPreviewDir(), str(segments[1])+".png")
			if os.path.exists(tp):
				return static.File(tp)
			else:
				raise svcs.UnknownURI("No such page: %s"%segments[1])
		elif name==b'edit':
			return DexterEdit(self.sourceName, segments[1])
		elif name==b'img':
			return ImagePiece(self.sourceName, segments[1])
		elif name==b'send':
			return SavedResult(self.sourceName, segments[1])
		elif name==b'receive':
			return ResultReceiver(self.sourceName, segments[1])
		raise svcs.UnknownURI("No such action: %s"%name)

		return None

	loader = template.TagLoader(T.html[
		T.head[
			T.title["DfyD: Graph Selection"],
			T.transparent(render="commonhead"),
		],
		T.body[
			T.div(style="position:fixed;top:0pt;right:0pt;background:white")[
				T.a(href="/dexter/ui/ui/info")["Service Info"]],
			T.h1["Dexter: Select graph/page"],
			NT(T.div, data="pages")(class_="previews",
					render="sequence")[
				NT(T.a, pattern="item")(render="imagelink"),
			],
			NT(T.div, data="extracted")(class_="dataPresent",
					render="ifdata")[
				T.h2["Data previously extracted"],
				T.ul(render="sequence")[
					NT(T.li, pattern="item")[
						T.a(render="resultlink"),
						" ",
						T.a(render="removelink")["[Delete]"]]]],
			T.p["You can upload a single image (in jpeg, png or gif formats)"
				" or a pdf file.  Please"
				" note that your images and data might be located by others if"
				" they know"
				" the name you gave your data set.  Delete your data set after"
				" use if this is not what you want."],
			T.transparent(render="form upload"),
			T.p[T.a(href="purgeData", render="childLink")[
				"Delete this data set from server"]],
		]
	])


class MainPage(formal.ResourceWithForm, web.ServiceBasedPage):
	name = "custom"

	@classmethod
	def isBrowseable(self, service):
		return True

	def form_enterName(self, request, data={}):
		form = formal.Form()
		form.addField("srcName", formal.String(required=True), label="Name",
			description="A freely choosable"
				" name for the data you want to work on")
		form.addAction(self.submitAction, label="Next")
		return form

	def crash(self, failure, request):
		return web.renderDCErrorPage(failure, request)

	def submitAction(self, request, form, data):
		srcName = data["srcName"].replace("/", "_")
		raise svcs.SeeOther(
			request.uri+("/"+(urllib.parse.quote(srcName))).encode("utf-8"))
	
	def getChild(self, name, request):
		return DocumentPage(name)

	loader = template.TagLoader(T.html[
		T.head[
			T.title["DfyD: Name Selection"],
			T.transparent(render="commonhead"),
		],
		T.body(render="withsidebar")[
			T.h1["Dexter: Choose Name"],
			T.p["Dexter is a service that allows you to reconstruct data"
				" from graphs you only have in PDF or as an image."],
			T.p["Before you can upload your source, please give your data some"
				" name.  You can use that name later to do further extraction,"
				" to revisit your work, or to pass it on to colleagues."],
			T.p["Dexter originally was a Java applet.  Modern browsers cannot"
				" exceute them any more, which is why we are using"
				" Leaningtech's ",
				T.a(href="https://www.leaningtech.com/cheerpj/")["CheerPJ"],
				" to get it executed using webassembly and javascript."
				" This works reasonably well in Chromium, less well in Firefox.  ",
				T.strong["Please be a bit patient at startup."]],
			T.p["If you run this more often, you may want to look into ",
				T.a(href="https://soft.g-vo.org/dexter")["running DaCHS locally"],
				"."],
			T.transparent(render="form enterName"),
			T.p(style="border: 1pt dashed red;width:60%; padding: 1ex;"
				"margin: 2ex auto")["Note: executing this will pull resources from ",
				T.a(href="https://www.leaningtech.com/")["Leaningtech"],
				" and run their code in your browser.  This is probably all right,"
				" but since it's a deviation of our ",
				T.a(href="/privpol.shtml")["privacy policy"],
				", we'd like to tell you up front."],
		]
	])