Too much (I tried to split it up but it won't work)

5 files changed, 344 insertions, 284 deletions

diff --git a/src/acit/__init__.py b/src/acit/__init__.py
index fd70d0d..318cfa7 100644
--- a/src/acit/__init__.py
+++ b/src/acit/__init__.py
@@ -5,20 +5,24 @@ from .web import Server
 
 from .imapplugin import ImapPlugin
 from .db import DBPoolManager
+from .pagegenerator import Generator
 
 from .types import Site
 
 def run():
 	db=DBPoolManager(cherrypy.engine)
 	db.subscribe()
-	site=Site(idbpool=db)
+	site=Site(dbpool=db)
 	imap=ImapPlugin(
 		cherrypy.engine, dbpool=db, site=site
 		)
 	imap.subscribe()
 
+	regen=Generator(cherrypy.engine,imap=imap,site=site,dbpool=db)
+	regen.subscribe()
 
-	server=Server(dbpool=db)
+
+	server=Server(dbpool=db,site=site)
 	cherrypy.quickstart(server)
 
 
diff --git a/src/acit/imapplugin.py b/src/acit/imapplugin.py
index f5a03a1..454904d 100644
--- a/src/acit/imapplugin.py
+++ b/src/acit/imapplugin.py
@@ -1,23 +1,20 @@
 
 import threading
 from os import getenv
-from queue import Queue, Empty
-
-from html import escape as html_escape
+from queue import Queue
 
 from cherrypy.process import plugins
 import cherrypy
 
 import re
 
-import imap_tools
 from imap_tools import MailBox, MailMessage
 
+from .imap_pool import MailBoxPool, PoolEmpty
+
 from .db import DBPoolManager
 from .types import Site
 
-from . import html
-
 class ImapPlugin(plugins.SimplePlugin):
 	def __init__(self,bus,dbpool:DBPoolManager,site:Site):
 		plugins.SimplePlugin.__init__(self,bus)
@@ -27,7 +24,7 @@ class ImapPlugin(plugins.SimplePlugin):
 		self.imap_server=getenv("ACIT_IMAP_SERVER")
 		self.imap_user=getenv("ACIT_IMAP_USER")
 		self.imap_pass=getenv("ACIT_IMAP_PASS")
-		self.imap_port=getenv("ACIT_IMAP_PORT",0)
+		self.imap_port=getenv("ACIT_IMAP_PORT",993)
 
 		self.smtp_server=getenv("ACIT_SMTP_SERVER")
 		self.smtp_user=getenv("ACIT_SMTP_USER")
@@ -38,24 +35,30 @@ class ImapPlugin(plugins.SimplePlugin):
 			raise ValueError("Missing ACIT_IMAP_SERVER, ACIT_IMAP_USER, ACIT_SMTP_SERVER, or ACIT_SMTP_USER")
 		# end block
 
+		self.mlog("IMAP config: %s @ %s : %d"%(self.imap_user,self.imap_server,self.imap_port))
+
 		# block: make storage attributes
 		self.mailin_thread=None
 		self.mailout_thread=None
-		self.regen_worker_thread=None
 		self.stopping=threading.Event()
 
 		self.mailout_queue=Queue()
-		self.regen_queue=Queue()
 		self.mailbox_per_thread={}
 		# end block
+
+		self.mbpool=MailBoxPool(
+			host=self.imap_server,port=self.imap_port,username=self.imap_user,password=self.imap_pass,
+			connection_n=int(getenv("ACIT_IMAP_POOL_SIZE",4))
+			)
+
+		self.index_lock=threading.Lock()
 	
 	def start(self):
-		self.mlog("Starting email-related loops, doing setup")
+		#self.mlog("Starting email-related loops, doing setup") # no need to log for quick stuff like this
 		# block: prepare threads
 		self.stopping.clear()
 		self.mailin_thread=threading.Thread(target=self.imap_loop_controller)
 		self.mailout_thread=threading.Thread(target=self.smtp_loop)
-		self.regen_worker_thread=threading.Thread(target=self.regen_worker)
 		# end block
 
 		# block: fetch configuration around email address usage from env
@@ -73,51 +76,79 @@ class ImapPlugin(plugins.SimplePlugin):
 			self.addr_format=self.emailname+"+{proj}#{bug}@"+self.emaildomain
 			self.addr_regex=self.emailname+r"+[^@#]*(#[0-9]*)?@"+self.emaildomain.replace(".","\\.")
 		# end block
+
+		self.mlog("Starting mailbox pool. This can take a while.")
+		self.mbpool.open()
+		pool=self.mbpool.get_pool_size()
+		if not pool:
+			self.mlog("Failed to put anything in the pool (%d)."%pool)
+			self.mlog("Errors:")
+			for e in self.mbpool.errors:
+				self.mlog(e)
+
+			self.mlog("\n\nNOTE!!! THIS WILL MAKE ACIT UNUSABLE, BECAUSE IT WON'T BE ABLE TO INTERACT WITH EMAIL.\n"
+				 " Please check your IMAP configuration.\n")
+
+		if not self.dbpool.poolStartedEvent.is_set():
+			cherrypy.engine.subscribe("db-started",self.update_index)
+		else:
+			self.update_index()
+
 	
 		self.mlog("Starting threads")
 		self.mailin_thread.start()
-		self.regen_worker_thread.start()
-		threading.Thread(target=self.discover_projects).start()
 		#self.mailout_thread.start()
 
 		self.mlog("Done")
 	
-	def discover_projects(self):
-		"Responsible for running through the imap structure, and queueing page generation for all bugs and projects"
-		threading.current_thread().setName("discovery")
-		self.mlog("Enlisting all currently known bugs for page generation...")
-		with self.get_MailBox() as mailbox:
-			
-			mailbox.folder.set(self.ensurefolder(mailbox,"bugs")) # cd into bugs folder
-			for folder in mailbox.folder.list("bugs"): # iterate over all subfolders
-				
-				# block: fetch bugname from foldername
-				path=folder.name.split(folder.delim)
-				if len(path)==3:
-					try:
-						proj=path[1]
-						bug=int(path[2])
-					except ValueError: # if bug isn't a number, int() raises ValueError
-						continue
-					# end block
+	def update_index(self,folders:list[str,int]=None):
+		"Note folder is a list of tuples consisting of trackername, bugid"
+		with self.dbpool.get_connection() as conn, conn.cursor() as cur, self.get_MailBox() as mb, self.index_lock:
+			cur.execute("CREATE TABLE IF NOT EXISTS msgindex ("
+				"tracker VARCHAR(80),"
+				"bugid INT,"
+				"messageid TINYTEXT UNIQUE"
+				")"
+				)
 
-					#self.mlog("Found %s#%d"%(proj,bug))
-					self.regen_queue.put((proj,bug),block=True) # enqueue update
+			if not folders:
+				self.mlog("Full-updating message index...")
+				cur.execute("SELECT tracker,bugid FROM bugs")
+				folders=[ (tracker, bugid) for tracker,bugid in cur ]
+
+			for tracker,bugid in folders:
+				try:
+					#self.mlog("Updating index for %s/%d"%(tracker,bugid))
+					mb.folder.set(self.get_bug_folder(mb,tracker,bugid))
+					for msg in mb.fetch():
+						if "message-id" in msg.headers:
+							cur.execute("REPLACE INTO msgindex VALUES (?,?,?)",(tracker,bugid,msg.headers["message-id"][:255]))
+
+				except Exception as e:
+					self.mlog("Error while indexing mailbox of %s/%d: %s"%(tracker,bugid,e))
+
+			conn.commit()
+			self.mlog("Updated message index")
+	
+	def find_in_reply_to(self,messageid):
+		with self.dbpool.get_connection() as conn, conn.cursor() as cur:
+			cur.execute("SELECT tracker,bugid FROM msgindex WHERE messageid=? LIMIT 1",(messageid,))
+			return cur.fetchone()
+	
+	def format_emailaddr(self,project,bugid=None,subject=None):
+		email=self.addr_format.format(proj=project, bug=bugid)
+		email=email.replace("#None",'')
+		if subject:
+			from urllib.parse import quote as quote
+			email+='?subject='
+			email+=quote(subject, safe='')
+		return email
 
-		self.mlog("Done")
 
 	def get_full_projectname(self,proj):
-		projects=["cgit","acit","folder","subfolder/folder", "whohoo/test","public/test"]
-		matches=[]
-		for project in projects:
-			if proj==project:
-				matches.clear()
-				matches.append(project)
-				break
-			else:
-				if ('/'+project).endswith(proj):
-					matches.append(proj)
-		return matches
+		"returns results from ``self.site.findtrackers()`` without modification"
+		return self.site.findtrackers(proj)
+
 
 	def stripInfoFromMailAddr(self,address:str):
 		"matches bugnumber and projectname from email address"
@@ -152,238 +183,136 @@ class ImapPlugin(plugins.SimplePlugin):
 			mailbox.folder.subscribe(fname,True)
 		return fname
 	
-	def get_bug_folder(self,mailbox:MailBox,proj,bug):
+	def get_bug_folder(self,mailbox:MailBox,proj,bug=None):
 		"helper to format the path to a bug's folder and ensure it's existence"
-		return self.ensurefolder(mailbox,"bugs",proj,str(bug))
+		path=["bugs"]
+		path.extend(proj.split('/'))
+		if bug: path.append(str(bug))
+		return self.ensurefolder(mailbox,*path)
 	
 	def get_MailBox(self):
-		"get a new mailbox, though only allows one mailbox per thread."
-		#tid=threading.current_thread()
-		#if not tid in self.mailbox_per_thread:
-		#	self.mailbox_per_thread[tid]=MailBox(self.imap_server).login(self.imap_user,self.imap_pass, None)
-		#return self.mailbox_per_thread[tid]
-		return MailBox(self.imap_server).login(self.imap_user,self.imap_pass, None)
-
-
-	def regen_worker(self):
-		"""
-		Listens to a queue and calls the page regenerator when requested via the queue.
-		Use by putting a tuple `(project,bugid)` in :attr:`regen_queue`
-		"""
-		threading.current_thread().setName("PageGenerator")
-		with self.get_MailBox() as mailbox:
-			dolog=True
-			while not self.stopping.is_set():
-				if dolog and self.regen_queue.empty():
-					self.mlog("Ready, waiting for items to enter queue...")
-				try:
-					proj,bug=self.regen_queue.get(timeout=1)
-				except Empty:
-					dolog=False
-					continue
-				self.mlog("Starting regen for %s/%s"%(proj or "",bug or ""))
-				self.generate_page(mailbox,proj,bug)
-				dolog=True
-
+		"get a new mailbox connection from the pool"
+		return self.mbpool.get_box()
 
-	def generate_page(self,mailbox:MailBox,project=None,bug=None):
-		"Responsible for regenerating pages. Only generates bug pages, forwards any request to generate project pages to that function."
-		if not bug and not project:
+	def imap_magic(self):
+		if self.stopping.wait(5): # if not stopping, this just times out after x seconds, so this is a nice timer
 			return
-		if project and not bug:
-			return self.generate_project_page(mailbox,project)
-
-		try:
-			# cd into mailbox folder
-			mailbox.folder.set(self.get_bug_folder(mailbox,project,bug))
-			pagelimit=25 # confusingly sets the limit on how many emails per page.
-			currentpage=""
-			pagenr=0
-			emailsonpage=0
-			og_message=None
-
-			from .util import (
-				lookahead, # this allows us to detect when we reached the last message in the mailbox
-				email2html
-				)
-
-			for msg, last in lookahead(mailbox.fetch()):
-				# block: allow showing original message & subject everywhere by storing it
-				if pagenr==0 and emailsonpage==0: # first email
-					og_message=msg
-				else:
-					# show an expandable thing with the original thing at the top of every page
-					currentpage+='<details class="ogmail"><summary>Click to expand original report</summary>'
-					currentpage+=html_escape(og_message.text)
-					currentpage+='</details>'
-				# end block
-
-				# block: page generation
-				#currentpage+='<section class="email">'
-				#currentpage+=html_escape(msg.text)
-				#currentpage+='</section><br>'
-				currentpage+=email2html(msg.text)
-				# end block
-				emailsonpage+=1
-
-				# block: complete page formatting, register page
-				if emailsonpage>=pagelimit or last:
-					resultingpage=html.mailpage.format(
-							project=project,
-							bug=bug,
-							subject=og_message.subject if og_message else "<undefined>",
-							content=currentpage,
-							pagenr=pagenr
-							)
-
-					# subblock: register page
-					path="%s / %d / %d"%(project,bug,pagenr)
-					cherrypy.engine.publish(
-						"newpage",
-						path=path,
-						content=resultingpage
-						)
-					# also register without page number if first page
-					if pagenr==0:
-						cherrypy.engine.publish(
-							"newpage",
-							path="%s/%d"%(project,bug),
-							content=resultingpage
-							)
-					# end subblock
-
-					# reset page-specific trackers
-					currentpage=""
-					emailsonpage=0
-					pagenr+=1
-				# end block
-
-		except Exception:
-			self.mlog("Error occured generating bug page for %s/%d"%(project,bug),traceback=True)
-
-	def generate_project_page(self,mailbox,proj):
-		return
+		refreshable={}
+		with self.get_MailBox() as mailbox:
+			mailbox.folder.set("INBOX")
 
-	
-	def imap_magic(self,mailbox:MailBox):
-		mailbox.folder.set("INBOX")
+			if mailbox.folder.status()["MESSAGES"]>0:
 
-		# block: wait 5 minutes and poll after that
-		mailbox.idle.start()
+				for msg in mailbox.fetch():
+					self.mlog("Processing email with subject '%s'"%msg.subject)
 
-		if self.stopping.wait(5): # if not stopping, this just times out after 300 seconds, so this is a nice timer
-			mailbox.idle.stop()
-			return
 
-		responses=mailbox.idle.poll(timeout=1)
-		mailbox.idle.stop()
-		# end block
+					for addr in msg.to + msg.cc + msg.bcc + msg.reply_to:
+						if re.fullmatch(self.addr_regex,addr):
+							proj,bug=self.stripInfoFromMailAddr(addr)
+							break
+					else:
+						proj=None
+						bug=None
 
-		if responses or mailbox.folder.status()["MESSAGES"]>0:
-			refreshable={}
+					if "in-reply-to" in msg.headers:
+						self.mlog("Using In-Reply-To header to figure out meta")
+						replyid=msg.headers["in-reply-to"]
+						data=self.find_in_reply_to(replyid)
+						if data:
+							proj,bug=data
 
-			for msg in mailbox.fetch():
-				self.mlog("Processing email with subject '%s'"%msg.subject)
+					# block: make sure a project was specified
+					if not proj:
+						self.mlog("No project specified.")
+						self.mail_error(msg,"Please specify a project by mailing to:\n "+\
+								("" if self.uses_aliases else self.emailname+"+")+"PROJECT@"+self.emaildomain+\
+								"\nwhere PROJECT is the name of your target project")
+						self.move_errored_mail(mailbox,msg)
+						continue
+					# end block
 
-				for addr in msg.to + msg.cc + msg.bcc + msg.reply_to:
-					if re.fullmatch(self.addr_regex,addr):
-						proj,bug=self.stripInfoFromMailAddr(addr)
-						break
-				else:
-					proj=None
-					bug=None
+					# block: make sure project exists
+					proj_matches=self.get_full_projectname(proj)
+					if not proj_matches:
+						self.mlog("Received email for nonexistent project %s"%proj)
+						self.mail_error(msg,notice="Project '%s' doesn't exist"%proj)
+						self.move_errored_mail(mailbox,msg)
+						continue
+					# end block
 
-				# block: make sure a project was specified
-				if not proj:
-					self.mlog("No project specified.")
-					self.mail_error(msg,"Please specify a project by mailing to:\n "+\
-							("" if self.uses_aliases else self.emailname+"+")+"PROJECT@"+self.emaildomain+\
-							"\nwhere PROJECT is the name of your target project")
-					self.move_errored_mail(mailbox,msg)
-					continue
-				# end block
+					# block: make sure only 1 project matches
+					if len(proj_matches)>1:
+						self.mlog("Conficting projectname. Sending projectlist.")
+						self.mail_error(msg,notice="Multiple projects found to match your query. Please specify. Options:\n%s"%"\n".join(proj_matches))
+						self.move_errored_mail(mailbox,msg)
+						continue
 
-				# block: make sure project exists
-				proj_matches=self.get_full_projectname(proj)
-				if not proj_matches:
-					self.mlog("Received email for nonexistent project %s"%proj)
-					self.mail_error(msg,notice="Project '%s' doesn't exist"%proj)
-					self.move_errored_mail(mailbox,msg)
-					continue
-				# end block
+					proj=proj_matches[0]
+					# end block
 
-				# block: make sure only 1 project matches
-				if len(proj_matches)>1:
-					self.mlog("Conficting projectname. Sending projectlist.")
-					self.mail_error(msg,notice="Multiple projects found to match your query. Please specify. Options:\n%s"%"\n".join(proj_matches))
-					self.move_errored_mail(mailbox,msg)
-					continue
+					# block: parse bug id
+					if not bug:
+						if re.match(r"^\[PATCH.*\]",msg.subject):
+							bugtype="PATCH"
+						elif re.match(r"^\[DISCUSSION.*\]",msg.subject):
+							bugtype="DISCUS"
+						else:
+							bugtype="BUG"
+						bug=self.site.newbug(proj,bugtype=bugtype)
+						bug.subject=msg.subject[:1024]
+						bug.description=\
+							'No description written.\nFirst email in thread:\n\n'+msg.text[:65535] # TODO: don't thruncate silently, send error to user.
+						self.mlog("Assigned new bugnr %d to '%s'"%(bug.bugid,msg.subject))
+						bug=bug.bugid
 
-				proj=proj_matches[0]
-				# end block
+					try:
+						bug=int(bug)
+					except ValueError as e:
+						self.mlog("Error decoding value to int:",e,traceback=True)
 
-				# block: parse bug id
-				if not bug:
-					if re.match(r"^\[PATCH.*\]",msg.subject):
-						bugtype="PATCH"
-					elif re.match(r"^\[DISCUSSION.*\]"):
-						bugtype="DISCUS"
-					else:
-						bugtype="BUG"
-					bug=self.site.newbug(proj,bugtype=bugtype)
-					bug.subject=msg.subject[:1024]
-					bug.description=msg.text[:65535] # TODO: don't thruncate silently, send error to user.
-					self.mlog("Assigned new bugnr %d to '%s'"%(bug,msg.subject))
+						self.mail_error(msg,notice="Exception while trying to convert bug number to integer",exception=e)
+						self.move_errored_mail(mailbox,msg)
+						continue
+					# end block
+					
+					# block: move mail to folder specific for this project/bug
+					try:
+						path=self.get_bug_folder(mailbox,proj,bug)
+						mailbox.move([msg.uid], path)
+						refreshable.setdefault(proj,[]).append(bug)
+					except Exception:
+						self.mlog("Error processing email '%s' for %s/%d"%(msg.subject,proj,bug),traceback=True)
+					# end block
 
-				try:
-					bug=int(bug)
-				except ValueError as e:
-					self.mlog("Error decoding value to int:",e,traceback=True)
+				# block: update all webpages that received new mail
+				for proj,bugs in refreshable.items():
+					# project page needs to be regenerated too (counters)
+					cherrypy.engine.publish("regen",proj,None)
+					for bug in bugs:
+						cherrypy.engine.publish("regen",proj,bug)
 
-					self.mail_error(msg,notice="Exception while trying to convert bug number to integer",exception=e)
-					self.move_errored_mail(mailbox,msg)
-					continue
 				# end block
-				
-				# block: move mail to folder specific for this project/bug
-				try:
-					path=self.get_bug_folder(mailbox,proj,bug)
-					mailbox.move([msg.uid], path)
-					refreshable.setdefault(proj,[]).append(bug)
-				except Exception:
-					self.mlog("Error processing email '%s' for %s/%d"%(msg.subject,proj,bug),traceback=True)
-				# end block
-
-			# block: update all webpages that received new mail
-			for proj,bugs in refreshable.items():
-				# project page needs to be regenerated too (counters)
-				self.regen_queue.put((proj,None),block=True)
-				for bug in bugs:
-					self.regen_queue.put((proj,bug),block=True)
 
-			# end block
+		if refreshable:
+			self.update_index([proj,bug] for bug in bugs for proj,bugs in  refreshable.items())
 
 	
 	def imap_loop_controller(self):
 		"Responsible for running imap_magic() repeatedly, and handling its errors."
 		threading.current_thread().setName("IMAPrunner")
-		self.mlog("Connecting to IMAP")
-		try:
-			with self.get_MailBox() as mailbox:
-				self.mlog("IMAP monitor thread started (connected)")
-				self.ensurefolder(mailbox,"INBOX")
-				while not self.stopping.is_set():
-					try:
-						self.imap_magic(mailbox)
-					except Exception:
-						import traceback
-						exc=traceback.format_exc()
-						self.mlog("Exception occured:\n%s"%exc)
-						self.mlog("!! this may lead to emails not appearing or appearing later !!")
-
-		except imap_tools.errors.MailboxLoginError:
-			self.mlog("Error logging in.")
-			cherrypy.engine.exit()
+		self.mlog("IMAP monitor thread started.")
+		while not self.stopping.is_set():
+			try:
+				self.imap_magic()
+			except PoolEmpty:
+				self.mlog("IMAP pool empty, unable to continue.")
+				break
+			except Exception:
+				import traceback
+				exc=traceback.format_exc()
+				self.mlog("Exception occured:\n%s"%exc)
+				self.mlog("!! this may lead to emails not appearing or appearing later !!")
 
 		self.mlog("IMAP monitor thread stopped.")
 	
@@ -403,9 +332,12 @@ class ImapPlugin(plugins.SimplePlugin):
 		"Sets stopping signal, waits for all threads to stop"
 		self.mlog("Stopping. This can take a while.")
 		self.stopping.set()
-		for thread in ( self.mailin_thread, self.mailout_thread, self.regen_worker_thread ):
+		for thread in ( self.mailin_thread, self.mailout_thread ):
 			if thread.is_alive() and not thread==threading.current_thread():
+				self.mlog("Waiting for thread: %s"%thread.name)
 				thread.join()
+		self.mlog("Closing IMAP pool")
+		self.mbpool.close()
 		self.mlog("Stopped")
 	
 	def mlog(self,*msg,**kwargs):
diff --git a/src/acit/types.py b/src/acit/types.py
index 2719590..4c84f26 100644
--- a/src/acit/types.py
+++ b/src/acit/types.py
@@ -1,8 +1,10 @@
 
-from .db import DBPoolManager
+from threading import RLock
+from datetime import datetime, timedelta
 
 import cherrypy
 
+from .db import DBPoolManager
 
 BUGSTATUS=["OPEN", "CLOSED", "UNCONF", "REJECT", "UPSTRM"]
 BUGTYPES=["BUG", "DISCUS", "PATCH"]
@@ -13,15 +15,16 @@ class Site():
 		cherrypy.engine.subscribe("db-started",self.on_db_connect)
 
 		self.bugcache={}
+		self.last_tracker_update=datetime.fromtimestamp(0)
+		self.tracker_update_lock=RLock()
 	
 	def on_db_connect(self):
+		cherrypy.log("Setting up database",context="SITE")
 		with self.dbpool.get_connection() as conn, conn.cursor() as cur:
 			cur.execute(
 				"CREATE TABLE IF NOT EXISTS trackers ("
-				"name VARCHAR(80),"
-				"category VARCHAR(80),"
-				"homepage VARCHAR(1024),"
-				"readme_url VARCHAR(1024)"
+				"name VARCHAR(80) PRIMARY KEY,"
+				"homepage VARCHAR(1024)"
 				")"
 				)
 
@@ -44,7 +47,77 @@ class Site():
 				")"
 				)
 
-	def getbug(self,tracker:str,bugid):
+			#cur.execute(
+			#	"CREATE TABLE IF NOT EXISTS accounts ("
+			#	"email VARCHAR(80),"
+			#	"hash VARCHAR(80),"
+			# 	)# TODO
+			
+			conn.commit()
+
+			self.update_all_bugpages(conn,cur)
+	
+
+	def update_all_bugpages(self,conn,cur,trackerfilter=None):
+		if trackerfilter:
+			cherrypy.engine.publish("regen",trackerfilter,None)
+		else:
+			cur.execute("SELECT name FROM trackers")
+			for [tracker] in cur:
+				cherrypy.engine.publish("regen",tracker,None)
+
+		cur.execute("SELECT tracker,bugid FROM bugs")
+		for [tracker,bugid] in cur:
+			if trackerfilter and not trackerfilter==tracker:
+				continue
+			cherrypy.engine.publish("regen",tracker,bugid)
+
+
+	def update_trackers(self,force=False):
+		from os import getenv
+		import subprocess
+
+		def log(msg,traceback=False):
+			cherrypy.log(msg=msg,traceback=traceback,context="REFRESH")
+
+		with self.tracker_update_lock:
+			if not force and self.last_tracker_update+timedelta(minutes=5)>datetime.now():
+				return
+			log("Updating trackerlist")
+			try:
+				with self.dbpool.get_connection() as conn, conn.cursor() as cur:
+					script=getenv("ACIT_LIST_TRACKERS","/usr/lib/acit-list-trackers")
+					proc=subprocess.run(script,capture_output=True)
+
+					if proc.stderr:
+						log("Refresh script generated STDERR:")
+						log(proc.stderr)
+
+					rows=proc.stdout.decode().split("\n")
+					for row in rows:
+						if row: # skip empty lines
+							data=row.split("\t")
+
+							if not len(data)==2:
+								log("Error processing line:\n> %s\nWeird value count (expected 2)."%row)
+								continue
+
+							name,homepage=data
+
+							log(name)
+
+							cur.execute("REPLACE INTO trackers (name,homepage) VALUES (?,?)",(name,homepage))
+							conn.commit()
+
+							self.update_all_bugpages(conn,cur,trackerfilter=name)
+
+					self.last_tracker_update=datetime.now()
+
+			except Exception:
+				cherrypy.log("Error refreshing trackers using script.",traceback=True,context="SITE")
+			
+
+	def getbug(self,tracker:str,bugid:int):
 		# make sure bug exists
 		with self.dbpool.get_connection() as conn, conn.cursor() as cur:
 			cur.execute("SELECT 1 FROM bugs WHERE tracker=? AND bugid=? LIMIT 1",(tracker,bugid))
@@ -64,17 +137,34 @@ class Site():
 		if not bugtype in BUGTYPES:
 			raise ValueError("Bugtype illegal")
 		with self.dbpool.get_connection() as conn, conn.cursor() as cur:
-			cur.execute("SELECT TOP 1 bugid FROM bugs WHERE tracker=? SORT BY bugid DESC",(tracker,))
+			cur.execute("SELECT bugid FROM bugs WHERE tracker=? ORDER BY bugid DESC LIMIT 1",(tracker,))
 			nr=cur.fetchone()
 			if not nr:
 				nr=0
+			else:
+				nr=nr[0]
 
 			nr+=1
-			cur.execute("INSERT INTO bugs VALUES (?,?)",(tracker,nr,"","","OPEN",bugtype))
+			cur.execute("INSERT INTO bugs VALUES (?,?,?,?,?,?)",(tracker,nr,"","","OPEN",bugtype))
+			conn.commit()
 
 		return self.getbug(tracker=tracker,bugid=nr)
 
-	
+	def findtrackers(self,query:str):
+		self.update_trackers()
+		with self.dbpool.get_connection() as conn, conn.cursor() as cur:
+			cur.execute("SELECT name FROM trackers")
+			trackers:list[str]=[ value[0] for value in cur.fetchall() ]
+			results=[]
+			for tracker in trackers:
+				if tracker==query:
+					return [tracker]
+				else:
+					if tracker.endswith(query):
+						results.append(tracker)
+			return results
+
+
 
 class Bug():
 	def __init__(self,site:Site,tracker:str,bugid:int):
@@ -82,7 +172,7 @@ class Bug():
 		self.tracker=tracker
 		self.bugid=bugid
 		with self.site.dbpool.get_connection() as conn, conn.cursor() as cur:
-			cur.execute("SELECT (subject,description,status,type) FROM bugs WHERE tracker=? AND bugid=? LIMIT 1",(tracker,bugid))
+			cur.execute("SELECT subject,description,status,type FROM bugs WHERE tracker=? AND bugid=? LIMIT 1",(tracker,bugid))
 			data=cur.fetchone()
 			if not data:
 				raise ValueError("Bug %s#%d does not exists!"%(tracker,bugid))
@@ -112,6 +202,7 @@ class Bug():
 
 		with self.site.dbpool.get_connection() as conn, conn.cursor() as cur:
 			cur.execute("UPDATE bugs SET subject=? WHERE tracker=? AND bugid=?",(value,self.tracker,self.bugid))
+			conn.commit()
 			self._cache["subject"]=value
 
 	@property
@@ -126,6 +217,7 @@ class Bug():
 
 		with self.site.dbpool.get_connection() as conn, conn.cursor() as cur:
 			cur.execute("UPDATE bugs SET description=? WHERE tracker=? AND bugid=?",(value,self.tracker,self.bugid))
+			conn.commit()
 			self._cache["description"]=value
 
 	@property
@@ -140,6 +232,7 @@ class Bug():
 
 		with self.site.dbpool.get_connection() as conn, conn.cursor() as cur:
 			cur.execute("UPDATE bugs SET status=? WHERE tracker=? AND bugid=?",(value,self.tracker,self.bugid))
+			conn.commit()
 			self._cache["status"]=value
 
 	@property
@@ -154,6 +247,7 @@ class Bug():
 
 		with self.site.dbpool.get_connection() as conn, conn.cursor() as cur:
 			cur.execute("UPDATE bugs SET type=? WHERE tracker=? AND bugid=?",(value,self.tracker,self.bugid))
+			conn.commit()
 			self._cache["type"]=value
 	
 	@property
@@ -164,15 +258,18 @@ class Bug():
 		"""
 		with self.site.dbpool.get_connection() as conn, conn.cursor() as cur:
 			cur.execute("SELECT email FROM subscribers WHERE tracker=? AND bugid=?",(self.tracker,self.bugid))
+			conn.commit()
 			return [ value[0] for value in cur ]
 	
 	def addsubscriber(self,email):
 		with self.site.dbpool.get_connection() as conn, conn.cursor() as cur:
 			cur.execute("INSERT INTO subscribers VALUES (?,?,?)",(self.tracker,self.bugid,email))
+			conn.commit()
 
 	def rmsubscriber(self,email):
 		with self.site.dbpool.get_connection() as conn, conn.cursor() as cur:
 			cur.execute("DELETE FROM subscribers WHERE tracker=? AND bugid=? AND email=?)",(self.tracker,self.bugid,email))
+			conn.commit()
 
 
 class Tracker():
@@ -180,7 +277,7 @@ class Tracker():
 		self.site=site
 		self.tracker=tracker
 		with self.site.dbpool.get_connection() as conn, conn.cursor() as cur:
-			cur.execute("SELECT (name,category,homepage,readme) FROM trackers WHERE name=? LIMIT 1",(tracker,))
+			cur.execute("SELECT name,homepage FROM trackers WHERE name=? LIMIT 1",(tracker,))
 			data=cur.fetchone()
 			if not data:
 				raise ValueError("Tracker %s does not exists!"%tracker)
@@ -195,18 +292,9 @@ class Tracker():
 	# NOTE: this is a lot of repeated code, maybe we can rewrite this into a __setattr__ and __getattribute__ method?
 	@property
 	def name(self):
-		"The name of this tracker"
+		"The name of this tracker. readonly, changing a tracker's name randomly seriously messes with the design."
 		return self._cache["name"]
 
-	@name.setter
-	def name(self,value):
-		if len(value)>80:
-			raise ValueError("Subject length not allowed to be higher than 80, this is %d"%len(value))
-
-		with self.site.dbpool.get_connection() as conn, conn.cursor() as cur:
-			cur.execute("UPDATE trackers SET name=? WHERE tracker=? AND bugid=?",(value,self.tracker,self.bugid))
-			self._cache["name"]=value
-
 	@property
 	def category(self):
 		"The category of this tracker"
@@ -219,6 +307,7 @@ class Tracker():
 
 		with self.site.dbpool.get_connection() as conn, conn.cursor() as cur:
 			cur.execute("UPDATE trackers SET category=? WHERE tracker=? AND bugid=?",(value,self.tracker,self.bugid))
+			conn.commit()
 			self._cache["category"]=value
 
 	@property
@@ -233,6 +322,7 @@ class Tracker():
 
 		with self.site.dbpool.get_connection() as conn, conn.cursor() as cur:
 			cur.execute("UPDATE trackers SET homepage=? WHERE tracker=? AND bugid=?",(value,self.tracker,self.bugid))
+			conn.commit()
 			self._cache["homepage"]=value
 
 	@property
@@ -247,4 +337,5 @@ class Tracker():
 
 		with self.site.dbpool.get_connection() as conn, conn.cursor() as cur:
 			cur.execute("UPDATE trackers SET readme_url=? WHERE tracker=? AND bugid=?",(value,self.tracker,self.bugid))
+			conn.commit()
 			self._cache["readme_url"]=value
diff --git a/src/acit/util.py b/src/acit/util.py
index e4e346c..7e89e6d 100644
--- a/src/acit/util.py
+++ b/src/acit/util.py
@@ -18,7 +18,7 @@ def lookahead(iterable): # https://stackoverflow.com/a/1630350
 	yield last, True
 
 def email2html(mailtext:str,extraclasses=""):
-	print(mailtext)
+	#print(mailtext)
 	from html import escape
 	res='<section class="email %s"><p>'%extraclasses
 	mail=escape(mailtext).replace("\n\n","</p><p>").replace("\n","<br>")
diff --git a/src/acit/web.py b/src/acit/web.py
index e09c94b..b6fb83b 100644
--- a/src/acit/web.py
+++ b/src/acit/web.py
@@ -2,34 +2,67 @@
 import cherrypy
 
 from .db import DBPoolManager
+from .types import Site
 
 class Server():
-	def __init__(self,dbpool:DBPoolManager):
+	def __init__(self,dbpool:DBPoolManager,site:Site):
 		cherrypy.engine.subscribe("newpage",self.registerpage)
-		self.pages={}
 		self.dbpool=dbpool
+		self.site=site
 	
-	def registerpage(self,path,content):
-		self.pages[path]=content
+	def registerpage(self,path,content,regentoken=None):
+		with self.dbpool.get_connection() as conn, conn.cursor() as cur:
+			cur.execute("CREATE TABLE IF NOT EXISTS pagecache ("
+				"path VARCHAR(100) PRIMARY KEY,"
+				"content MEDIUMTEXT,"
+				"generate VARCHAR(40)"
+				")")
+			cur.execute("INSERT INTO pagecache VALUES (?,?,?) ON DUPLICATE KEY UPDATE content=?, generate=?",(path,content,regentoken,content,regentoken))
+			conn.commit()
+	
+	def getpage(self,path,**kwargs):
+		with self.dbpool.get_connection() as conn, conn.cursor() as cur:
+			cur.execute("SELECT content,generate FROM pagecache WHERE path=? LIMIT 1",(path,))
+			data=cur.fetchone()
+			if not data:
+				return
+			page,generate=data
+			if kwargs:
+				results=cherrypy.engine.publish('generate::'+generate,path=path,**kwargs)
+				if results:
+					cherrypy.log(results)
+					return results[0]
+			else:
+				return page
+			
 
 	@cherrypy.expose
 	def index(self,quote="nothing"):
-		return "Index"
+		import threading
+		self.site.update_trackers()
+		cherrypy.log("Listing threads:")
+		for thread in threading.enumerate():
+			cherrypy.log(repr(thread))
+		
+		from os import getenv
+		raise cherrypy.HTTPRedirect(getenv("ACIT_HOME_REDIRECT","about:blank"),303)
 	
 	@cherrypy.expose(["style.css"])
 	def style(self):
 		from .html import style_css
+		cherrypy.response.headers["Content-Type"]="text/css"
 		return style_css
 
 	@cherrypy.expose
 	def default(self,*pathlist,**kwargs):
 		from os.path import normpath
 		path=normpath("/".join(pathlist))
-		if path in self.pages:
-			return self.pages[path]
-		else:
+		page=self.getpage(path)
+		if not page:
 			from .html import notfound
 			cherrypy.response.status=404
 			return notfound.format(path=path)
+		else:
+			return page