from cherrypy.process import plugins import cherrypy from queue import Queue, Empty import threading from collections import Counter, defaultdict from imap_tools.errors import MailboxFolderSelectError from .imapplugin import ImapPlugin from .types import Site from .db import DBPoolManager from . import html class Generator(plugins.SimplePlugin): def __init__(self,bus,imap:ImapPlugin,site:Site,dbpool:DBPoolManager): plugins.SimplePlugin.__init__(self,bus) self.imap=imap self.site=site self.dbpool=dbpool self.queue=Queue() self.stopping=threading.Event() self.thread=None cherrypy.engine.subscribe("regen",self.enqueue_update) cherrypy.engine.subscribe("generate::projectpage",self.generate_project_page_dynamically) self.statcache:dict[str,Counter]=defaultdict(lambda:Counter()) def enqueue_update(self,project,bugid): self.queue.put((project,bugid),block=True) def start(self): self.mlog("Starting...") self.thread=threading.Thread(target=self.worker) self.thread.start() def worker(self): """ Listens to a queue and calls the page regenerator when requested via the queue. Use by putting a tuple `(project,bugid)` in :attr:`queue` """ threading.current_thread().setName("PageGenerator") dolog=True while not self.stopping.is_set(): if dolog and self.queue.empty(): self.mlog("Ready, waiting for items to enter queue...") try: proj,bug=self.queue.get(timeout=1) except Empty: dolog=False continue #self.mlog("Starting regen for %s/%s"%(proj or "",bug or "")) self.generate_page(project=proj,bug=bug) dolog=True def generate_page(self,project=None,bug=None): "Responsible for regenerating pages. Only generates bug pages, forwards any request to generate project pages to that function." if not bug and not project: return if project and not bug: return self.generate_project_page(project) try: bugobj=self.site.getbug(tracker=project,bugid=bug) with self.imap.get_MailBox() as mailbox: # cd into mailbox folder mailbox.folder.set(self.imap.get_bug_folder(mailbox,project,bug)) pagelimit=25 # confusingly sets the limit on how many emails per page. currentpage="" pagenr=1 emailsonpage=0 from .util import ( lookahead, # this allows us to detect when we reached the last message in the mailbox email2html ) totalemails=mailbox.folder.status()["MESSAGES"] # only used for displaying to user and providing a `last page` link #totalemails=30*totalemails # again, for testing purposes totalpages=totalemails//pagelimit + min(totalemails%pagelimit,1) self.mlog("Generating %d pages from %d emails for %s/%d"%(totalpages,totalemails,project,bug)) #mails=[i for i in mailbox.fetch()]*30 # DON'T UNCOMMENT, for testing purposes. # when using above line, change ``mailbox.fetch()`` below for ``mails`` for msg, last in lookahead(mailbox.fetch()): # block: allow showing bug description # show an expandable thing with the original thing at the top of every page if emailsonpage==0: currentpage+='
=2: pageswitcher+='<<first | '%(project,bug) if pagenr>=3: pageswitcher+='<prev | '%(project,bug,pagenr-1) pageswitcher+="[page %d/%d]"%(pagenr,totalpages) if totalpages-pagenr>1: # if more than 2 pages left pageswitcher+=' | next>'%(project,bug,pagenr+1) if totalpages-pagenr>0: # if more than 1 pages left pageswitcher+=' | last>>'%(project,bug,totalpages) # block: complete page formatting, register page if emailsonpage>=pagelimit or last: #first=(pagenr-1)*25 #jumptos="| jump: "+", ".join([ # '#%d'%(i,i) for i in range(first, first+emailsonpage+1) # ]) resultingpage=html.mailpage.format( emailaddr=self.imap.format_emailaddr(project,bug,subject="RE: "+bugobj.subject), subscribe=self.imap.format_emailaddr(project,bug,subject="SUBSCRIBE"), status=bugobj.status, bugtype=bugobj.type, project=project, bug=bug, subject=bugobj.subject, content=currentpage, pagenr=pagenr, pageswitcher=pageswitcher, jumptos="" ) # subblock: register page path="%s/%d/%d"%(project,bug,pagenr) cherrypy.engine.publish( "newpage", path=path, content=resultingpage ) # also register without page number if first page if pagenr==1: cherrypy.engine.publish( "newpage", path="%s/%d"%(project,bug), content=resultingpage ) # end subblock # reset page-specific trackers currentpage="" emailsonpage=0 pagenr+=1 # end block except Exception: self.mlog("Error occured generating bug page for %s/%d"%(project,bug),traceback=True) def generate_project_page_dynamically(self,path='',**kwargs): if not path: return return self.generate_project_page(mailbox=None,proj=path,register=False,**kwargs) def generate_project_page(self,proj,register=True,**kwargs): from .html import projectpage from datetime import datetime, timedelta statc=self.statcache[proj] if register: statc.clear() self.mlog("Generating projectpage %s"%proj) if register: with self.imap.get_MailBox() as mailbox: self.mlog("Updating statcounter for %s"%proj) for folder in mailbox.folder.list(self.imap.get_bug_folder(mailbox,proj)): try: mailbox.folder.set(folder.name) for email in mailbox.fetch(): statc["total"]+=1 if email.date>datetime.now(tz=email.date.tzinfo)-timedelta(hours=24): statc["today"]+=1 #self.mlog("%d: %s"%(statc["today"],email.subject)) except MailboxFolderSelectError: pass with self.dbpool.get_connection() as conn, conn.cursor() as cur: args=[proj] extraquery='' if kwargs and not register: for bugtype in ["bug","patch","discus"]: if bugtype in kwargs: kwargs.setdefault("types",[]).append(bugtype.upper()) for status in ["open","closed","reject","upstrm","unconf"]: if status in kwargs: kwargs.setdefault("status",[]).append(status.upper()) if 'subject' in kwargs: extraquery+="AND LOWER(subject) LIKE LOWER(?)" args.append("%"+kwargs["subject"]+"%") if 'types' in kwargs: extraquery+="AND type IN ("+",".join("?"*len(kwargs["types"]))+")" args.extend(kwargs["types"]) if 'status' in kwargs: extraquery+="AND status IN ("+",".join("?"*len(kwargs["status"]))+")" args.extend(kwargs["status"]) filter_subject=kwargs.get("subject","") filter_by=kwargs.get("by","") def default_kws(*keys): for key in keys: if key in kwargs: break else: for key in keys: kwargs[key]="true" default_kws("bug","patch","discus") filter_bug=kwargs.get("bug") filter_patch=kwargs.get("patch") filter_discus=kwargs.get("discus") default_kws("open","closed","reject","upstrm","unconf") filter_open=kwargs.get("open") filter_closed=kwargs.get("closed") filter_reject=kwargs.get("reject") filter_upstrm=kwargs.get("upstrm") filter_unconf=kwargs.get("unconf") sql="SELECT bugid,subject,status,type FROM bugs WHERE tracker=? "+\ extraquery+\ " ORDER BY FIELD(status,'OPEN') DESC" #self.mlog(sql,args) cur.execute(sql,args) tabledata="" for id,subject,status,bugtype in cur: if register: statc[bugtype+status]+=1 statc[bugtype]+=1 tabledata+='' tabledata+='%d'%id tabledata+='%s'%(proj,id,subject) tabledata+='{status}'.format(status=status) tabledata+='%s'%bugtype tabledata+='' result=projectpage.format( project=proj, emailaddr=self.imap.format_emailaddr(project=proj), readme="TODO", mailtoday=statc["today"],mailtotal=statc["total"], bugstotal=statc["BUG"], bugsopen=statc["BUGOPEN"], bugsclosed=statc["BUGCLOSED"], patchestotal=statc["PATCH"], patchesopen=statc["PATCHOPEN"], patchesclosed=statc["PATCHCLOSED"], patchesrejected=statc["PATCHREJECT"], filter_subject=filter_subject, filter_by=filter_by, filter_bug="checked" if filter_bug else "", filter_patch="checked" if filter_patch else "", filter_discus="checked" if filter_discus else "", filter_open="checked" if filter_open else "", filter_closed="checked" if filter_closed else "", filter_reject="checked" if filter_reject else "", filter_upstrm="checked" if filter_upstrm else "", filter_unconf="checked" if filter_unconf else "", maillist=tabledata ) if register: cherrypy.engine.publish( "newpage", path=proj, content=result, regentoken="projectpage" ) else: return result def stop(self): if not self.thread or not self.thread.is_alive(): return self.mlog("Signalling stop to page generator") self.stopping.set() self.thread.join() self.mlog("Stopped") def mlog(self,*msg,**kwargs): import traceback function=traceback.extract_stack(limit=2)[0].name cherrypy.log(context="%s>>REGEN:%s"%(threading.current_thread().name, function), msg=" ".join([str(i) for i in msg]),**kwargs)