path: root/src/acit/imapplugin.py

import threading
from os import getenv
from queue import Queue

from cherrypy.process import plugins
import cherrypy

import re

from imap_tools import MailBox, MailMessage

from .imap_pool import MailBoxPool, PoolEmpty

from .db import DBPoolManager
from .types import Site

class ImapPlugin(plugins.SimplePlugin):
	def __init__(self,bus,dbpool:DBPoolManager,site:Site):
		plugins.SimplePlugin.__init__(self,bus)
		self.dbpool=dbpool
		self.site=site
		# block: get configuration variables from env
		self.imap_server=getenv("ACIT_IMAP_SERVER")
		self.imap_user=getenv("ACIT_IMAP_USER")
		self.imap_pass=getenv("ACIT_IMAP_PASS")
		self.imap_port=int(getenv("ACIT_IMAP_PORT",993))

		self.smtp_server=getenv("ACIT_SMTP_SERVER")
		self.smtp_user=getenv("ACIT_SMTP_USER")
		self.smtp_pass=getenv("ACIT_SMTP_PASS")
		self.smtp_port=int(getenv("ACIT_SMTP_PORT",0))

		if not ( self.imap_server and self.imap_user and self.imap_pass and self.smtp_server and self.smtp_user and self.smtp_pass ):
			raise ValueError("Missing ACIT_IMAP_SERVER, ACIT_IMAP_USER, ACIT_SMTP_SERVER, or ACIT_SMTP_USER")
		# end block

		self.mlog("IMAP config: %s @ %s : %d"%(self.imap_user,self.imap_server,self.imap_port))

		# block: make storage attributes
		self.mailin_thread=None
		self.mailout_thread=None
		self.stopping=threading.Event()

		self.mailout_queue=Queue()
		self.mailbox_per_thread={}
		# end block

		self.mbpool=MailBoxPool(
			host=self.imap_server,port=self.imap_port,username=self.imap_user,password=self.imap_pass,
			connection_n=int(getenv("ACIT_IMAP_POOL_SIZE",4))
			)

		self.index_lock=threading.Lock()
	
	def start(self):
		#self.mlog("Starting email-related loops, doing setup") # no need to log for quick stuff like this
		# block: prepare threads
		self.stopping.clear()
		self.mailin_thread=threading.Thread(target=self.imap_loop_controller)
		self.mailout_thread=threading.Thread(target=self.smtp_loop)
		# end block

		# block: fetch configuration around email address usage from env
		self.uses_aliases=getenv("ACIT_MAIL_USES_ALIASES",False)

		name,domain=self.imap_user.rsplit("@",1)
		self.emaildomain=getenv("ACIT_MAIL_DOMAIN",domain)
		self.emailname=getenv("ACIT_MAIL_NAME",name)

		if self.uses_aliases: # if we use aliases, we match project#bug@example.com
			self.addr_format="{proj}#{bug}@"+self.emaildomain
			self.addr_regex="[^@#]*(#[0-9]*)?@"+ self.emaildomain.replace(".","\\.")

		else: # if we use plus addresses, we match name+project#bug@example.com
			self.addr_format=self.emailname+"+{proj}#{bug}@"+self.emaildomain
			self.addr_regex=self.emailname+r"+[^@#]*(#[0-9]*)?@"+self.emaildomain.replace(".","\\.")
		# end block

		self.mlog("Starting mailbox pool. This can take a while.")
		self.mbpool.open()
		pool=self.mbpool.get_pool_size()
		if not pool:
			self.mlog("Failed to put anything in the pool (%d)."%pool)
			self.mlog("Errors:")
			import traceback
			for e in self.mbpool.errors:
				tb=traceback.format_exception(e)
				self.mlog(repr(e),tb)

			self.mlog("\n\nNOTE!!! THIS WILL MAKE ACIT UNUSABLE, BECAUSE IT WON'T BE ABLE TO INTERACT WITH EMAIL.\n"
				 " Please check your IMAP configuration.\n")

		if not self.dbpool.poolStartedEvent.is_set():
			cherrypy.engine.subscribe("db-started",self.update_index)
		else:
			self.update_index()

	
		self.mlog("Starting threads")
		self.mailin_thread.start()
		#self.mailout_thread.start()

		self.mlog("Done")
	
	def update_index(self,folders:list[str,int]=None):
		"Note folder is a list of tuples consisting of trackername, bugid"
		with self.dbpool.get_connection() as conn, conn.cursor() as cur, self.get_MailBox() as mb, self.index_lock:
			cur.execute("CREATE TABLE IF NOT EXISTS msgindex ("
				"tracker VARCHAR(80),"
				"bugid INT,"
				"messageid TINYTEXT UNIQUE"
				")"
				)

			if not folders:
				self.mlog("Full-updating message index...")
				cur.execute("SELECT tracker,bugid FROM bugs")
				folders=[ (tracker, bugid) for tracker,bugid in cur ]

			for tracker,bugid in folders:
				try:
					#self.mlog("Updating index for %s/%d"%(tracker,bugid))
					mb.folder.set(self.get_bug_folder(mb,tracker,bugid))
					for msg in mb.fetch():
						if "message-id" in msg.headers:
							cur.execute("REPLACE INTO msgindex VALUES (?,?,?)",(tracker,bugid,msg.headers["message-id"][:255]))

				except Exception as e:
					self.mlog("Error while indexing mailbox of %s/%d: %s"%(tracker,bugid,e))

			conn.commit()
			self.mlog("Updated message index")
	
	def find_in_reply_to(self,messageid):
		with self.dbpool.get_connection() as conn, conn.cursor() as cur:
			cur.execute("SELECT tracker,bugid FROM msgindex WHERE messageid=? LIMIT 1",(messageid,))
			return cur.fetchone()
	
	def format_emailaddr(self,project,bugid=None,subject=None):
		email=self.addr_format.format(proj=project, bug=bugid)
		email=email.replace("#None",'')
		if subject:
			from urllib.parse import quote as quote
			email+='?subject='
			email+=quote(subject, safe='')
		return email


	def get_full_projectname(self,proj):
		"returns results from ``self.site.findtrackers()`` without modification"
		return self.site.findtrackers(proj)


	def stripInfoFromMailAddr(self,address:str):
		"matches bugnumber and projectname from email address"
		addr=address.removesuffix("@"+self.emaildomain)

		if not self.uses_aliases:
			if not '+' in addr:
				return (None,None)

			addr=addr.removeprefix(self.emailname)
			addr=addr.removeprefix("+")

		
		if '#' in addr:
			proj,bug=addr.rsplit("#",1)
		else:
			proj=addr
			bug=None

		return (proj,bug)

	def ensurefolder(self,mailbox:MailBox,*path): 
		"makes sure a folder exists on the mailserver"
		# assuming from rfc3501 section 7.2.2:
		# > All children of a top-level hierarchy node MUST use
      # > the same separator character.
		# hoping the whole mailserver uses the same delimiter
		delim=mailbox.folder.list('')[0].delim
		fname=delim.join([ str(i) for i in path])
		if not mailbox.folder.exists(fname):
			mailbox.folder.create(fname)
			mailbox.folder.subscribe(fname,True)
		return fname
	
	def get_bug_folder(self,mailbox:MailBox,proj,bug=None):
		"helper to format the path to a bug's folder and ensure it's existence"
		path=["bugs"]
		path.extend(proj.split('/'))
		if bug: path.append(str(bug))
		return self.ensurefolder(mailbox,*path)
	
	def get_MailBox(self):
		"get a new mailbox connection from the pool"
		return self.mbpool.get_box()

	def imap_magic(self):
		if self.stopping.wait(5): # if not stopping, this just times out after x seconds, so this is a nice timer
			return
		refreshable={}
		with self.get_MailBox() as mailbox:
			mailbox.folder.set("INBOX")

			if mailbox.folder.status()["MESSAGES"]>0:

				for msg in mailbox.fetch():
					target=self.handle_email(mailbox,msg)
					if target:
						proj,bug=target
						refreshable.setdefault(proj,[]).append(bug)

		# block: update all webpages that received new mail
		for proj,bugs in refreshable.items():
			# project page needs to be regenerated too (counters)
			cherrypy.engine.publish("regen",proj,None)
			for bug in bugs:
				cherrypy.engine.publish("regen",proj,bug)

		# end block

		if refreshable:
			self.update_index([proj,bug] for bug in bugs for proj,bugs in  refreshable.items())

	def handle_email(self,mailbox:MailBox,msg:MailMessage):
		self.mlog("Processing email with subject '%s'"%msg.subject)


		for addr in msg.to + msg.cc + msg.bcc + msg.reply_to:
			if re.fullmatch(self.addr_regex,addr):
				proj,bug=self.stripInfoFromMailAddr(addr)
				break
		else:
			proj=None
			bug=None

		if "in-reply-to" in msg.headers:
			self.mlog("Using In-Reply-To header to figure out meta")
			replyid=msg.headers["in-reply-to"]
			data=self.find_in_reply_to(replyid)
			if data:
				proj,bug=data
			else: # try again later, maybe we need to index first or handle some other email first
				self.mlog("Message-ID not in index, trying again next round")
				return

		# block: make sure a project was specified
		if not proj:
			self.mlog("No project specified.")
			self.mail_error(msg,"Please specify a project by mailing to:\n "+\
					("" if self.uses_aliases else self.emailname+"+")+"PROJECT@"+self.emaildomain+\
					"\nwhere PROJECT is the name of your target project")
			self.move_errored_mail(mailbox,msg)
			return
		# end block

		# block: make sure project exists
		proj_matches=self.get_full_projectname(proj)
		if not proj_matches:
			self.mlog("Received email for nonexistent project %s"%proj)
			self.mail_error(msg,notice="Project '%s' doesn't exist"%proj)
			self.move_errored_mail(mailbox,msg)
			return
		# end block

		# block: make sure only 1 project matches
		if len(proj_matches)>1:
			self.mlog("Conficting projectname. Sending projectlist.")
			self.mail_error(msg,notice="Multiple projects found to match your query. Please specify. Options:\n%s"%"\n".join(proj_matches))
			self.move_errored_mail(mailbox,msg)
			return

		proj=proj_matches[0]
		# end block

		# block: parse bug id
		if not bug:
			if re.match(r"^\[PATCH.*\]",msg.subject):
				bugtype="PATCH"
			elif re.match(r"^\[DISCUSSION.*\]",msg.subject):
				bugtype="DISCUS"
			else:
				bugtype="BUG"
			bug=self.site.newbug(proj,bugtype=bugtype)
			bug.subject=msg.subject[:1024]
			bug.description=\
				'No description written.\nFirst email in thread:\n\n'+msg.text[:65535] # TODO: don't thruncate silently, send error to user.
			self.mlog("Assigned new bugnr %d to '%s'"%(bug.bugid,msg.subject))
			bug=bug.bugid

		try:
			bug=int(bug)
		except ValueError as e:
			self.mlog("Error decoding value to int:",e,traceback=True)

			self.mail_error(msg,notice="Exception while trying to convert bug number to integer",exception=e)
			self.move_errored_mail(mailbox,msg)
			return
		# end block
		
		# block: move mail to folder specific for this project/bug
		self.mlog("Email '%s' into %s/%d"%(msg.subject,proj,bug))
		try:
			path=self.get_bug_folder(mailbox,proj,bug)
			mailbox.move([msg.uid], path)
		except Exception:
			self.mlog("Error processing email '%s' for %s/%d"%(msg.subject,proj,bug),traceback=True)
		# end block
		
		return (proj,bug)
	
	def imap_loop_controller(self):
		"Responsible for running imap_magic() repeatedly, and handling its errors."
		threading.current_thread().setName("IMAPrunner")
		self.mlog("IMAP monitor thread started.")
		while not self.stopping.is_set():
			try:
				self.imap_magic()
			except PoolEmpty:
				self.mlog("IMAP pool empty, unable to continue.")
				break
			except Exception:
				import traceback
				exc=traceback.format_exc()
				self.mlog("Exception occured:\n%s"%exc)
				self.mlog("!! this may lead to emails not appearing or appearing later !!")

		self.mlog("IMAP monitor thread stopped.")
	
	def mail_error(self,msg:MailMessage,notice:str=None,exception:Exception=None):
		pass

	def move_errored_mail(self,mailbox:MailBox,msg:MailMessage,):
		target=self.ensurefolder(mailbox,"INBOX","Errors")
		return mailbox.move(msg.uid,target)
		


	def smtp_loop(self):
		pass
	
	def stop(self):
		"Sets stopping signal, waits for all threads to stop"
		self.mlog("Stopping. This can take a while.")
		self.stopping.set()
		for thread in ( self.mailin_thread, self.mailout_thread ):
			if thread.is_alive() and not thread==threading.current_thread():
				self.mlog("Waiting for thread: %s"%thread.name)
				thread.join()
		self.mlog("Closing IMAP pool")
		self.mbpool.close()
		self.mlog("Stopped")
	
	def mlog(self,*msg,**kwargs):
		import traceback
		function=traceback.extract_stack(limit=2)[0].name
		cherrypy.log(context="%s>>MAIL:%s"%(threading.current_thread().name, function), msg=" ".join([str(i) for i in msg]),**kwargs)