add dedicated code for generating webpages

1 files changed, 283 insertions, 0 deletions

diff --git a/src/acit/pagegenerator.py b/src/acit/pagegenerator.py
new file mode 100644
index 0000000..a75eb84
--- /dev/null
+++ b/src/acit/pagegenerator.py
@@ -0,0 +1,283 @@
+
+from cherrypy.process import plugins
+import cherrypy
+
+from queue import Queue, Empty
+import threading
+from collections import Counter, defaultdict
+
+from imap_tools import MailBox
+from imap_tools.errors import MailboxFolderSelectError
+
+from .imapplugin import ImapPlugin
+from .types import Site
+from .db import DBPoolManager
+
+from . import html
+
+class Generator(plugins.SimplePlugin):
+	def __init__(self,bus,imap:ImapPlugin,site:Site,dbpool:DBPoolManager):
+		plugins.SimplePlugin.__init__(self,bus)
+
+		self.imap=imap
+		self.site=site
+		self.dbpool=dbpool
+
+		self.queue=Queue()
+		self.stopping=threading.Event()
+		self.thread=None
+
+		cherrypy.engine.subscribe("regen",self.enqueue_update)
+		cherrypy.engine.subscribe("generate::projectpage",self.generate_project_page_dynamically)
+
+		self.statcache=defaultdict(lambda:Counter())
+
+	def enqueue_update(self,project,bugid):
+		self.queue.put((project,bugid),block=True)
+
+	def start(self):
+		self.mlog("Starting...")
+		self.thread=threading.Thread(target=self.worker)
+		self.thread.start()
+	
+	
+	def worker(self):
+		"""
+		Listens to a queue and calls the page regenerator when requested via the queue.
+		Use by putting a tuple `(project,bugid)` in :attr:`queue`
+		"""
+		threading.current_thread().setName("PageGenerator")
+		dolog=True
+		while not self.stopping.is_set():
+			if dolog and self.queue.empty():
+				self.mlog("Ready, waiting for items to enter queue...")
+			try:
+				proj,bug=self.queue.get(timeout=1)
+			except Empty:
+				dolog=False
+				continue
+
+			#self.mlog("Starting regen for %s/%s"%(proj or "",bug or ""))
+			self.generate_page(project=proj,bug=bug)
+			dolog=True
+
+
+	def generate_page(self,project=None,bug=None):
+		"Responsible for regenerating pages. Only generates bug pages, forwards any request to generate project pages to that function."
+		if not bug and not project:
+			return
+		if project and not bug:
+			return self.generate_project_page(project)
+
+		try:
+			bugobj=self.site.getbug(tracker=project,bugid=bug)
+			with self.imap.get_MailBox() as mailbox:
+				# cd into mailbox folder
+				mailbox.folder.set(self.imap.get_bug_folder(mailbox,project,bug))
+				pagelimit=25 # confusingly sets the limit on how many emails per page.
+				currentpage=""
+				pagenr=1
+				emailsonpage=0
+
+				from .util import (
+					lookahead, # this allows us to detect when we reached the last message in the mailbox
+					email2html
+					)
+
+				totalemails=mailbox.folder.status()["MESSAGES"] # only used for displaying to user and providing a `last page` link
+				#totalemails=30*totalemails # again, for testing purposes
+				totalpages=totalemails//pagelimit + min(totalemails%pagelimit,1)
+				self.mlog("Generating %d pages from %d emails for %s/%d"%(totalpages,totalemails,project,bug))
+
+				#mails=[i for i in mailbox.fetch()]*30 # DON'T UNCOMMENT, for testing purposes.
+				# when using above line, change ``mailbox.fetch()`` below for ``mails``
+
+				for msg, last in lookahead(mailbox.fetch()):
+					# block: allow showing bug description
+					# show an expandable thing with the original thing at the top of every page
+					if emailsonpage==0:
+						currentpage+='<details class="bugdesc"'
+						if pagenr==1:
+							currentpage+=' open'
+						currentpage+='><summary>Click to expand bug description</summary>'
+						currentpage+=email2html(bugobj.description)
+						currentpage+='</details>'
+					# end block
+
+					# block: page generation
+					currentpage+='<div class="emailheader" id="{n}"><a href="#{n}">#{n}</a> On {date}, {from_} wrote,<br>'.format(
+						n=(pagenr-1)*25 +emailsonpage,
+						date=msg.date_str,
+						from_=msg.from_)
+					currentpage+='with Subject: <i>%s</i></div>'%msg.subject
+					currentpage+=email2html(msg.text)
+					# end block
+					emailsonpage+=1
+
+					pageswitcher=''
+					if pagenr>=2:
+						pageswitcher+='<a href="/%s/%d/1">&lt;&lt;first</a> | '%(project,bug)
+					if pagenr>=3:
+						pageswitcher+='<a href="/%s/%d/%d">&lt;prev</a> | '%(project,bug,pagenr-1)
+					pageswitcher+="[page %d/%d]"%(pagenr,totalpages)
+					if totalpages-pagenr>1: # if more than 2 pages left
+						pageswitcher+=' | <a href="/%s/%d/%d">next&gt;</a>'%(project,bug,pagenr+1)
+					if totalpages-pagenr>0: # if more than 1 pages left
+						pageswitcher+=' | <a href="/%s/%d/%d">last&gt;&gt;</a>'%(project,bug,totalpages)
+
+					# block: complete page formatting, register page
+					if emailsonpage>=pagelimit or last:
+
+						#first=(pagenr-1)*25
+						#jumptos="| jump: "+", ".join([
+						#	'<a href="#%d">#%d</a>'%(i,i) for i in range(first, first+emailsonpage+1)
+						#	])
+
+
+						resultingpage=html.mailpage.format(
+								emailaddr=self.imap.format_emailaddr(project,bug,subject="RE: "+bugobj.subject),
+								subscribe=self.imap.format_emailaddr(project,bug,subject="SUBSCRIBE"),
+								status=bugobj.status,
+								bugtype=bugobj.type,
+								project=project,
+								bug=bug,
+								subject=bugobj.subject,
+								content=currentpage,
+								pagenr=pagenr,
+								pageswitcher=pageswitcher,
+								jumptos=""
+								)
+
+						# subblock: register page
+						path="%s/%d/%d"%(project,bug,pagenr)
+						cherrypy.engine.publish(
+							"newpage",
+							path=path,
+							content=resultingpage
+							)
+						# also register without page number if first page
+						if pagenr==1:
+							cherrypy.engine.publish(
+								"newpage",
+								path="%s/%d"%(project,bug),
+								content=resultingpage
+								)
+						# end subblock
+
+						# reset page-specific trackers
+						currentpage=""
+						emailsonpage=0
+						pagenr+=1
+					# end block
+
+		except Exception:
+			self.mlog("Error occured generating bug page for %s/%d"%(project,bug),traceback=True)
+	
+	def generate_project_page_dynamically(self,path='',**kwargs):
+		if not path:
+			return
+		return self.generate_project_page(mailbox=None,proj=path,register=False,**kwargs)
+
+	def generate_project_page(self,proj,register=True,**kwargs):
+		from .html import projectpage
+		from datetime import datetime, timedelta
+
+		statc=self.statcache[proj]
+		
+		if register:
+			with self.imap.get_MailBox() as mailbox:
+				self.mlog("Updating statcounter for %s"%proj)
+				for folder in mailbox.folder.list(self.imap.get_bug_folder(mailbox,proj)):
+					try:
+						mailbox.folder.set(folder.name)
+						for email in mailbox.fetch():
+							statc["total"]+=1
+							if email.date>datetime.now(tz=email.date.tzinfo)-timedelta(hours=24):
+								statc["today"]+=1
+							#self.mlog("%d: %s"%(statc["today"],email.subject))
+					except MailboxFolderSelectError:
+						pass
+				self.mlog("Updated mailbox statcounter")
+
+		with self.dbpool.get_connection() as conn, conn.cursor() as cur:
+			args=[proj]
+			extraquery=''
+			if kwargs:
+				if 'subject' in kwargs:
+					extraquery+="AND LOWER(subject) LIKE LOWER(?)"
+					args.append("%"+kwargs["subject"]+"%")
+				if 'types' in kwargs:
+					extraquery+="AND type IN ("+",".join("?"*len(kwargs["types"]))+")"
+					args.extend(kwargs["types"])
+				if 'status' in kwargs:
+					extraquery+="AND status IN ("+",".join("?"*len(kwargs["status"]))+")"
+					args.extend(kwargs["status"])
+
+			sql="SELECT bugid,subject,status,type FROM bugs WHERE tracker=? "+\
+				extraquery+\
+				" ORDER BY FIELD(status,'OPEN') DESC"
+			#self.mlog(sql,args)
+
+			cur.execute(sql,args)
+
+			tabledata=""
+
+			for id,subject,status,bugtype in cur:
+				if register:
+					statc[bugtype+status]+=1
+					statc[bugtype]+=1
+
+				tabledata+='<tr>'
+				tabledata+='<td class="bugnr" id="bugid">%d</td>'%id
+				tabledata+='<td id="subject"><a href="/%s/%d">%s</a></td>'%(proj,id,subject)
+				tabledata+='<td id="status" class=".status-{status}">{status}</td>'.format(status=status)
+				tabledata+='<td id="type">%s</td>'%bugtype
+				tabledata+='</tr>'
+
+
+
+		result=projectpage.format(
+			project=proj,
+			emailaddr=self.imap.format_emailaddr(project=proj),
+			readme="TODO",
+
+			mailtoday=statc["today"],mailtotal=statc["total"],
+
+			bugstotal=statc["BUG"],
+			bugsopen=statc["BUGOPEN"],
+			bugsclosed=statc["BUGCLOSED"],
+
+			patchestotal=statc["PATCH"],
+			patchesopen=statc["PATCHOPEN"],
+			patchesclosed=statc["PATCHCLOSED"],
+			patchesrejected=statc["PATCHREJECT"],
+
+			maillist=tabledata
+			)
+		if register:
+			cherrypy.engine.publish(
+				"newpage",
+				path=proj,
+				content=result,
+				regentoken="projectpage"
+				)
+		else:
+			return result
+	
+	def stop(self):
+		if not self.thread or not self.thread.is_alive():
+			return
+		self.mlog("Signalling stop to page generator")
+		self.stopping.set()
+		self.thread.join()
+		self.mlog("Stopped")
+
+	
+	def mlog(self,*msg,**kwargs):
+		import traceback
+		function=traceback.extract_stack(limit=2)[0].name
+		cherrypy.log(context="%s>>REGEN:%s"%(threading.current_thread().name, function), msg=" ".join([str(i) for i in msg]),**kwargs)
+
+
+
+