More IMAP magic: it now sorts email! Doesn't generate anything though :(

1 files changed, 186 insertions, 90 deletions

diff --git a/src/acit/imapplugin.py b/src/acit/imapplugin.py
index 56eda22..e15591a 100644
--- a/src/acit/imapplugin.py
+++ b/src/acit/imapplugin.py
@@ -10,6 +10,7 @@ import cherrypy
 
 import re
 
+import imap_tools
 from imap_tools import MailBox, MailMessage
 
 from typing import Literal
@@ -17,34 +18,35 @@ from typing import Literal
 from . import html
 
 class ImapPlugin(plugins.SimplePlugin):
-	def __init__(self,bus,
-				  imap_server:str,
-				  imap_user:str,
-				  imap_pass:str,
-				  smtp_server:str,
-				  smtp_user:str,
-				  smtp_pass:str,
-				  ):
+	def __init__(self,bus):
 		plugins.SimplePlugin.__init__(self,bus)
-		self.imap_server=imap_server
-		self.imap_user=imap_user
-		self.imap_pass=imap_pass
-		self.smtp_server=smtp_server
-		self.smtp_user=smtp_user
-		self.smtp_pass=smtp_pass
+		self.imap_server=getenv("ACIT_IMAP_SERVER")
+		self.imap_user=getenv("ACIT_IMAP_USER")
+		self.imap_pass=getenv("ACIT_IMAP_PASS")
+		self.imap_port=getenv("ACIT_IMAP_PORT",0)
+
+		self.smtp_server=getenv("ACIT_SMTP_SERVER")
+		self.smtp_user=getenv("ACIT_SMTP_USER")
+		self.smtp_pass=getenv("ACIT_SMTP_PASS")
+		self.smtp_port=getenv("ACIT_SMTP_PORT",0)
+
+		if not ( self.imap_server and self.imap_user and self.smtp_server and self.smtp_user ):
+			raise ValueError("Missing ACIT_IMAP_SERVER, ACIT_IMAP_USER, ACIT_SMTP_SERVER, or ACIT_SMTP_USER")
 
 		self.mailin_thread=None
 		self.mailout_thread=None
 		self.stopping=threading.Event()
 
 		self.mailout_queue=Queue()
+		self.mailbox_per_thread={}
 
 		emailcache={}
 	
 	def start(self):
+		self.mlog("Starting email-related loops, doing setup")
 		self.stopping.clear()
-		self.mailin_thread=threading.Thread(target=self.in_runner)
-		self.mailout_thread=threading.Thread(target=self.out_runner)
+		self.mailin_thread=threading.Thread(target=self.imap_loop_controller)
+		self.mailout_thread=threading.Thread(target=self.smtp_loop)
 
 		# the following commented block was too free to be parsed. If you can get it to work, feel free to email a patch.
 		#self.addrformat=getenv("ACIT_ADDRESSFORMAT",
@@ -64,30 +66,75 @@ class ImapPlugin(plugins.SimplePlugin):
 
 		if self.uses_aliases:
 			self.addr_format="{proj}#{bug}@"+self.emaildomain
-			self.addr_regex=r"[^@#\+]*#[0-9]*@"+ self.emaildomain.replace(".","\\.")
+			self.addr_regex="[^@#]*(#[0-9]*)?@"+ self.emaildomain.replace(".","\\.")
 
 		else:
 			self.addr_format=self.emailname+"+{proj}#{bug}@"+self.emaildomain
-			self.addr_regex=self.emailname+r"+[^@#\+]*#[0-9]*@"+self.emaildomain
+			self.addr_regex=self.emailname+r"+[^@#]*(#[0-9]*)?@"+self.emaildomain.replace(".","\\.")
 	
 
 
 
 		self.mailin_thread.start()
-		self.mailout_thread.start()
+		#self.mailout_thread.start()
 
-		self.mailbox_per_thread={}
+		self.mlog("Done")
+
+	def get_full_projectname(self,proj):
+		projects=["cgit","acit","folder","subfolder/folder"]
+		matches=[]
+		for project in projects:
+			if proj==project:
+				matches.clear()
+				matches.append(project)
+				break
+			else:
+				if ('/'+project).endswith(proj):
+					matches.append(proj)
+		return matches
 
 	def stripInfoFromMailAddr(self,address:str):
+		addr=address.removesuffix("@"+self.emaildomain)
+
 		if not self.uses_aliases:
-			address=address.removesuffix(self.emailname+"+")
+			if not '+' in addr:
+				return (None,None)
+
+			addr=addr.removeprefix(self.emailname)
+			addr=addr.removeprefix("+")
+
 		
-		address=address.removesuffix("@"+self.emaildomain)
-		return address.rsplit("#",1)
+		if '#' in addr:
+			proj,bug=addr.rsplit("#",1)
+		else:
+			proj=addr
+			bug=None
+
+		return (proj,bug)
 
+	def ensurefolder(self,mailbox:MailBox,*path):
+		# assuming from rfc3501 section 7.2.2:
+		# > All children of a top-level hierarchy node MUST use
+      # > the same separator character.
+		# hoping the whole mailserver uses the same delimiter
+		delim=mailbox.folder.list('')[0].delim
+		fname=delim.join([ str(i) for i in path])
+		if not mailbox.folder.exists(fname):
+			mailbox.folder.create(fname)
+			mailbox.folder.subscribe(fname,True)
+		return fname
 	
-	def get_bug_folder(self,proj,bug):
-		return self.ensurefolder("bugs",proj,str(bug))
+	def get_bug_folder(self,mailbox:MailBox,proj,bug):
+		return self.ensurefolder(mailbox,"bugs",proj,str(bug))
+	
+	def assign_new_bugnr(self,mailbox:MailBox,proj):
+		current=mailbox.folder.list(self.ensurefolder(mailbox,"bugs",proj))
+
+		nrs=[ folder.name.rsplit(folder.delim,1)[-1] for folder in current if folder.delim in folder.name ]
+		nrs=[ int(i) for i in nrs if i.isdigit()  ]
+		high=max([0]+nrs)
+		return high+1
+
 
 
 	def get_MailBox(self):
@@ -98,13 +145,14 @@ class ImapPlugin(plugins.SimplePlugin):
 
 
 	def generate_page(self,mailbox:MailBox,project=None,bug=None):
+		return
 		if not bug and not project:
 			return self.generate_main_page(mailbox)
 		if project and not bug:
 			return self.generate_project_page(mailbox,project)
 
 		try:
-			mailbox.folder.set(self.get_bug_folder(project,bug))
+			mailbox.folder.set(self.get_bug_folder(mailbox,project,bug))
 			pagelimit=25
 			currentpage=""
 			for msg in mailbox.fetch():
@@ -117,93 +165,141 @@ class ImapPlugin(plugins.SimplePlugin):
 
 
 	
-	def in_runner(self):
-		with self.get_MailBox() as mailbox:
-			self.mlog("IMAP monitor thread started (connected)")
-			while not self.stopping.is_set():
-				try:
-					mailbox.folder.set("INBOX")
+	def imap_magic(self,mailbox:MailBox):
+		mailbox.folder.set("INBOX")
 
-					# block: wait 5 minutes and poll after that
-					mailbox.idle.start()
+		# block: wait 5 minutes and poll after that
+		mailbox.idle.start()
 
-					if self.stopping.wait(300): # if not stopping, this just times out after 300 seconds, so this is a nice timer
-						mailbox.idle.stop()
-						break
+		if self.stopping.wait(5): # if not stopping, this just times out after 300 seconds, so this is a nice timer
+			mailbox.idle.stop()
+			return
 
-					responses=mailbox.idle.poll(timeout=1)
-					mailbox.idle.stop()
-					# end block
+		responses=mailbox.idle.poll(timeout=1)
+		mailbox.idle.stop()
+		# end block
 
-					if responses or mailbox.folder.status()["MESSAGES"]>0:
-						refreshable={}
+		if responses or mailbox.folder.status()["MESSAGES"]>0:
+			refreshable={}
 
-						for msg in mailbox.fetch():
-							for addr in msg.to + msg.cc + msg.bcc + msg.reply_to:
-								if re.fullmatch(self.addr_regex,addr):
-									proj,bug=self.stripInfoFromMailAddr(addr)
-									break
+			for msg in mailbox.fetch():
+				self.mlog("Processing email with subject '%s'"%msg.subject)
 
-							# block: parse bug id
-							try:
-								bug=int(bug)
-							except ValueError as e:
-								self.mlog("Error decoding value to int:",e,traceback=True)
+				for addr in msg.to + msg.cc + msg.bcc + msg.reply_to:
+					if re.fullmatch(self.addr_regex,addr):
+						proj,bug=self.stripInfoFromMailAddr(addr)
+						break
+				else:
+					proj=None
+					bug=None
+
+				# block: make sure a project was specified
+				if not proj:
+					self.mlog("No project specified.")
+					self.mail_error(msg,"Please specify a project by mailing to:\n "+\
+							("" if self.uses_aliases else self.emailname+"+")+"PROJECT@"+self.emaildomain+\
+							"\nwhere PROJECT is the name of your target project")
+					self.move_errored_mail(mailbox,msg)
+					continue
+				# end block
 
-								self.mail_error(msg,notice="Exception while trying to convert bug number to integer",exception=e)
-								self.move_errored_mail(msg)
-								continue
-							# end block
-							
-							# block: make sure project exists
-							proj=self.get_full_projectname(proj)
-							if not proj:
-								self.mlog("Received email for nonexistend project %s"%proj)
-								self.mail_error(msg,notice="Project '%s' doesn't exist"%proj)
-								self.move_errored_mail(msg)
-								continue
-							# end block
-							
-							# block: move mail to folder specific for this project/bug
-							try:
-								path=self.get_bug_folder(proj,bug)
-								mailbox.move([msg.uid], path)
-								refreshable.setdefault(proj,[]).append(bug)
-							except Exception:
-								self.mlog("Error processing email '%s' for %s/%d"%(msg.subject,proj,bug),traceback=True)
-							# end block
+				# block: make sure project exists
+				proj_matches=self.get_full_projectname(proj)
+				if not proj_matches:
+					self.mlog("Received email for nonexistent project %s"%proj)
+					self.mail_error(msg,notice="Project '%s' doesn't exist"%proj)
+					self.move_errored_mail(mailbox,msg)
+					continue
+				# end block
+				# block: make sure only 1 project matches
+				if len(proj_matches)>1:
+					self.mlog("Conficting projectname. Sending projectlist.")
+					self.mail_error(msg,notice="Multiple projects found to match your query. Please specify. Options:\n%s"%"\n".join(proj_matches))
+					self.move_errored_mail(mailbox,msg)
+					continue
 
-						# block: update all webpages that received new mail
-						for proj,bugs in refreshable.items():
-							self.generate_page(proj,None) # project page needs to be regenerated too (counters)
-							for bug in bugs:
-								self.generate_page(proj,bug)
+				proj=proj_matches[0]
 
-						self.generate_page(None,None) # main page needs to be generated too (a new project may have appeared + counters)
-						# end block
+				# block: parse bug id
+				if not bug:
+					bug=self.assign_new_bugnr(mailbox,proj)
+					self.mlog("Assigned new bugnr %d to '%s'"%(bug,msg.subject))
 
+				try:
+					bug=int(bug)
+				except ValueError as e:
+					self.mlog("Error decoding value to int:",e,traceback=True)
 
+					self.mail_error(msg,notice="Exception while trying to convert bug number to integer",exception=e)
+					self.move_errored_mail(mailbox,msg)
+					continue
+				# end block
+				
+				# block: move mail to folder specific for this project/bug
+				try:
+					path=self.get_bug_folder(mailbox,proj,bug)
+					mailbox.move([msg.uid], path)
+					refreshable.setdefault(proj,[]).append(bug)
 				except Exception:
-					import traceback
-					exc=traceback.format_exc()
-					self.mlog("Exception occured:\n%s"%exc)
-					self.mlog("!! this may lead to emails not appearing or appearing later !!")
+					self.mlog("Error processing email '%s' for %s/%d"%(msg.subject,proj,bug),traceback=True)
+				# end block
+
+			# block: update all webpages that received new mail
+			for proj,bugs in refreshable.items():
+				self.generate_page(proj,None) # project page needs to be regenerated too (counters)
+				for bug in bugs:
+					self.generate_page(proj,bug)
 
-			self.mlog("IMAP monitor thread stopped.")
+			self.generate_page(None,None) # main page needs to be generated too (a new project may have appeared + counters)
+			# end block
+
+	
+	def imap_loop_controller(self):
+		threading.current_thread().setName("IMAPrunner")
+		self.mlog("Connecting to IMAP")
+		try:
+			with self.get_MailBox() as mailbox:
+				self.mlog("IMAP monitor thread started (connected)")
+				self.ensurefolder(mailbox,"INBOX")
+				while not self.stopping.is_set():
+					try:
+						self.imap_magic(mailbox)
+					except Exception:
+						import traceback
+						exc=traceback.format_exc()
+						self.mlog("Exception occured:\n%s"%exc)
+						self.mlog("!! this may lead to emails not appearing or appearing later !!")
+
+		except imap_tools.errors.MailboxLoginError:
+			self.mlog("Error logging in.")
+			cherrypy.engine.exit()
+
+		self.mlog("IMAP monitor thread stopped.")
+	
+	def mail_error(self,msg:MailMessage,notice:str=None,exception:Exception=None):
+		pass
+
+	def move_errored_mail(self,mailbox:MailBox,msg:MailMessage,):
+		target=self.ensurefolder(mailbox,"INBOX","Errors")
+		return mailbox.move(msg.uid,target)
+		
 
 
-	def out_runner(self):
+	def smtp_loop(self):
 		pass
 	
 	def stop(self):
 		self.mlog("Stopping. This can take a while.")
 		self.stopping.set()
-		self.mailin_thread.join()
-		self.mailout_thread.join()
+		for thread in ( self.mailin_thread, self.mailout_thread ):
+			if thread.is_alive() and not thread==threading.current_thread():
+				thread.join()
 		self.mlog("Stopped")
 	
 	def mlog(self,*msg,**kwargs):
-		cherrypy.log(context="MAIL", msg=" ".join([str(i) for i in msg]),**kwargs)
+		import traceback
+		function=traceback.extract_stack(limit=2)[0].name
+		cherrypy.log(context="MAIL:%s:%s"%(threading.current_thread().name, function), msg=" ".join([str(i) for i in msg]),**kwargs)