from cherrypy.process import plugins
import cherrypy
from queue import Queue, Empty
import threading
from collections import Counter, defaultdict
from imap_tools.errors import MailboxFolderSelectError
from .imapplugin import ImapPlugin
from .types import Site
from .db import DBPoolManager
from . import html
class Generator(plugins.SimplePlugin):
def __init__(self,bus,imap:ImapPlugin,site:Site,dbpool:DBPoolManager):
plugins.SimplePlugin.__init__(self,bus)
self.imap=imap
self.site=site
self.dbpool=dbpool
self.queue=Queue()
self.stopping=threading.Event()
self.thread=None
cherrypy.engine.subscribe("regen",self.enqueue_update)
cherrypy.engine.subscribe("generate::projectpage",self.generate_project_page_dynamically)
self.statcache:dict[str,Counter]=defaultdict(lambda:Counter())
def enqueue_update(self,project,bugid):
self.queue.put((project,bugid),block=True)
def start(self):
self.mlog("Starting...")
self.thread=threading.Thread(target=self.worker)
self.thread.start()
def worker(self):
"""
Listens to a queue and calls the page regenerator when requested via the queue.
Use by putting a tuple `(project,bugid)` in :attr:`queue`
"""
threading.current_thread().setName("PageGenerator")
dolog=True
while not self.stopping.is_set():
if dolog and self.queue.empty():
self.mlog("Ready, waiting for items to enter queue...")
try:
proj,bug=self.queue.get(timeout=1)
except Empty:
dolog=False
continue
#self.mlog("Starting regen for %s/%s"%(proj or "",bug or ""))
self.generate_page(project=proj,bug=bug)
dolog=True
def generate_page(self,project=None,bug=None):
"Responsible for regenerating pages. Only generates bug pages, forwards any request to generate project pages to that function."
if not bug and not project:
return
if project and not bug:
return self.generate_project_page(project)
try:
bugobj=self.site.getbug(tracker=project,bugid=bug)
with self.imap.get_MailBox() as mailbox:
# cd into mailbox folder
mailbox.folder.set(self.imap.get_bug_folder(mailbox,project,bug))
pagelimit=25 # confusingly sets the limit on how many emails per page.
currentpage=""
pagenr=1
emailsonpage=0
from .util import (
lookahead, # this allows us to detect when we reached the last message in the mailbox
email2html
)
totalemails=mailbox.folder.status()["MESSAGES"] # only used for displaying to user and providing a `last page` link
#totalemails=30*totalemails # again, for testing purposes
totalpages=totalemails//pagelimit + min(totalemails%pagelimit,1)
self.mlog("Generating %d pages from %d emails for %s/%d"%(totalpages,totalemails,project,bug))
#mails=[i for i in mailbox.fetch()]*30 # DON'T UNCOMMENT, for testing purposes.
# when using above line, change ``mailbox.fetch()`` below for ``mails``
for msg, last in lookahead(mailbox.fetch()):
# block: allow showing bug description
# show an expandable thing with the original thing at the top of every page
if emailsonpage==0:
currentpage+='<details class="bugdesc"'
if pagenr==1:
currentpage+=' open'
currentpage+='><summary>Click to expand bug description</summary>'
currentpage+=email2html(bugobj.description)
currentpage+='</details>'
# end block
# block: page generation
currentpage+='<div class="emailheader" id="{n}"><a href="#{n}">#{n}</a> On {date}, {from_} wrote,<br>'.format(
n=(pagenr-1)*25 +emailsonpage,
date=msg.date_str,
from_=msg.from_)
currentpage+='with Subject: <i>%s</i></div>'%msg.subject
currentpage+=email2html(msg.text, ispatch=msg.subject.startswith("[PATCH"))
# end block
emailsonpage+=1
pageswitcher=''
if pagenr>=2:
pageswitcher+='<a href="/%s/%d/1"><<first</a> | '%(project,bug)
if pagenr>=3:
pageswitcher+='<a href="/%s/%d/%d"><prev</a> | '%(project,bug,pagenr-1)
pageswitcher+="[page %d/%d]"%(pagenr,totalpages)
if totalpages-pagenr>1: # if more than 2 pages left
pageswitcher+=' | <a href="/%s/%d/%d">next></a>'%(project,bug,pagenr+1)
if totalpages-pagenr>0: # if more than 1 pages left
pageswitcher+=' | <a href="/%s/%d/%d">last>></a>'%(project,bug,totalpages)
# block: complete page formatting, register page
if emailsonpage>=pagelimit or last:
#first=(pagenr-1)*25
#jumptos="| jump: "+", ".join([
# '<a href="#%d">#%d</a>'%(i,i) for i in range(first, first+emailsonpage+1)
# ])
resultingpage=html.mailpage.format(
emailaddr=self.imap.format_emailaddr(project,bug,subject="RE: "+bugobj.subject),
subscribe=self.imap.format_emailaddr(project,bug,subject="SUBSCRIBE"),
status=bugobj.status,
bugtype=bugobj.type,
project=project,
bug=bug,
subject=bugobj.subject,
content=currentpage,
pagenr=pagenr,
pageswitcher=pageswitcher,
jumptos=""
)
# subblock: register page
path="%s/%d/%d"%(project,bug,pagenr)
cherrypy.engine.publish(
"newpage",
path=path,
content=resultingpage
)
# also register without page number if first page
if pagenr==1:
cherrypy.engine.publish(
"newpage",
path="%s/%d"%(project,bug),
content=resultingpage
)
# end subblock
# reset page-specific trackers
currentpage=""
emailsonpage=0
pagenr+=1
# end block
except Exception:
self.mlog("Error occured generating bug page for %s/%d"%(project,bug),traceback=True)
def generate_project_page_dynamically(self,path='',**kwargs):
if not path:
return
return self.generate_project_page(mailbox=None,proj=path,register=False,**kwargs)
def generate_project_page(self,proj,register=True,**kwargs):
from .html import projectpage
from datetime import datetime, timedelta
statc=self.statcache[proj]
if register:
statc.clear()
self.mlog("Generating projectpage %s"%proj)
if register:
with self.imap.get_MailBox() as mailbox:
self.mlog("Updating statcounter for %s"%proj)
for folder in mailbox.folder.list(self.imap.get_bug_folder(mailbox,proj)):
try:
mailbox.folder.set(folder.name)
for email in mailbox.fetch():
statc["total"]+=1
if email.date>datetime.now(tz=email.date.tzinfo)-timedelta(hours=24):
statc["today"]+=1
#self.mlog("%d: %s"%(statc["today"],email.subject))
except MailboxFolderSelectError:
pass
with self.dbpool.get_connection() as conn, conn.cursor() as cur:
args=[proj]
extraquery=''
if kwargs and not register:
for bugtype in ["bug","patch","discus"]:
if bugtype in kwargs:
kwargs.setdefault("types",[]).append(bugtype.upper())
for status in ["open","closed","reject","upstrm","unconf"]:
if status in kwargs:
kwargs.setdefault("status",[]).append(status.upper())
if 'subject' in kwargs:
extraquery+="AND LOWER(subject) LIKE LOWER(?)"
args.append("%"+kwargs["subject"]+"%")
if 'types' in kwargs:
extraquery+="AND type IN ("+",".join("?"*len(kwargs["types"]))+")"
args.extend(kwargs["types"])
if 'status' in kwargs:
extraquery+="AND status IN ("+",".join("?"*len(kwargs["status"]))+")"
args.extend(kwargs["status"])
filter_subject=kwargs.get("subject","")
filter_by=kwargs.get("by","")
def default_kws(*keys):
for key in keys:
if key in kwargs:
break
else:
for key in keys:
kwargs[key]="true"
default_kws("bug","patch","discus")
filter_bug=kwargs.get("bug")
filter_patch=kwargs.get("patch")
filter_discus=kwargs.get("discus")
default_kws("open","closed","reject","upstrm","unconf")
filter_open=kwargs.get("open")
filter_closed=kwargs.get("closed")
filter_reject=kwargs.get("reject")
filter_upstrm=kwargs.get("upstrm")
filter_unconf=kwargs.get("unconf")
sql="SELECT bugid,subject,status,type FROM bugs WHERE tracker=? "+\
extraquery+\
" ORDER BY FIELD(status,'OPEN') DESC"
#self.mlog(sql,args)
cur.execute(sql,args)
tabledata=""
for id,subject,status,bugtype in cur:
if register:
statc[bugtype+status]+=1
statc[bugtype]+=1
tabledata+='<tr>'
tabledata+='<td class="bugnr" id="bugid">%d</td>'%id
tabledata+='<td id="subject"><a href="/%s/%d">%s</a></td>'%(proj,id,subject)
tabledata+='<td id="status" class="status-{status}">{status}</td>'.format(status=status)
tabledata+='<td id="type">%s</td>'%bugtype
tabledata+='</tr>'
result=projectpage.format(
project=proj,
emailaddr=self.imap.format_emailaddr(project=proj),
readme="TODO",
mailtoday=statc["today"],mailtotal=statc["total"],
bugstotal=statc["BUG"],
bugsopen=statc["BUGOPEN"],
bugsclosed=statc["BUGCLOSED"],
patchestotal=statc["PATCH"],
patchesopen=statc["PATCHOPEN"],
patchesclosed=statc["PATCHCLOSED"],
patchesrejected=statc["PATCHREJECT"],
filter_subject=filter_subject,
filter_by=filter_by,
filter_bug="checked" if filter_bug else "",
filter_patch="checked" if filter_patch else "",
filter_discus="checked" if filter_discus else "",
filter_open="checked" if filter_open else "",
filter_closed="checked" if filter_closed else "",
filter_reject="checked" if filter_reject else "",
filter_upstrm="checked" if filter_upstrm else "",
filter_unconf="checked" if filter_unconf else "",
maillist=tabledata
)
if register:
cherrypy.engine.publish(
"newpage",
path=proj,
content=result,
regentoken="projectpage"
)
else:
return result
def stop(self):
if not self.thread or not self.thread.is_alive():
return
self.mlog("Signalling stop to page generator")
self.stopping.set()
self.thread.join()
self.mlog("Stopped")
def mlog(self,*msg,**kwargs):
import traceback
function=traceback.extract_stack(limit=2)[0].name
cherrypy.log(context="%s>>REGEN:%s"%(threading.current_thread().name, function), msg=" ".join([str(i) for i in msg]),**kwargs)
