Email commands, plus 'a _bit_ more'

- you can now edit bugs via email
- fixed acit not resending mailinglist email correctly
- make sure you don't make a new bug when receiving the first email of a
  bug in your inbox and replying to it
- make forwarding email only delete loopback email when save argument is false
- don't add empty Cc or Bcc headers to prevent errors when sending
  without them
- make the thing that formats the In-Reply-To and References header a
  separate function
- only update projectpages when reloading projectlist, not bugpages
- fix tracker subscribers not working

4 files changed, 231 insertions, 36 deletions

diff --git a/src/acit/emailcommands.py b/src/acit/emailcommands.py
new file mode 100644
index 0000000..ff1ff5f
--- /dev/null
+++ b/src/acit/emailcommands.py
@@ -0,0 +1,16 @@
+
+import re
+
+from typing import Iterable
+
+def findcommands(mailtext:str) -> Iterable[re.Match]:
+	return re.finditer(r'\n(' # match any of:
+		r'STATUS (OPEN|CLOSED|UNCONF|REJECT|UPSTRM)|' # a status change, STATUS <state>
+		r'TYPE (BUG|DISCUS|PATCH)|' # a type change, TYPE <type>
+		r'SUBJECT .*|' # a subject change, SUBJECT <any valid text without newlines>
+		r'DESCRIPTION( @[0-9]*(:[0-9]*)?)?\r?\n(.*|\r?\n)*\r?\nDESCRIPTION END|' # DESCRIPTION [@<startingline[:<endingline]]\n<any valid text including newlines>\nEND DESCRIPTION
+		r'(UN)?PERMIT .*)' # change command permission for an emailaddress, (UN)PERMIT <email>
+		,mailtext)
+
+
+
diff --git a/src/acit/imapplugin.py b/src/acit/imapplugin.py
index a73a2de..9a6e8df 100644
--- a/src/acit/imapplugin.py
+++ b/src/acit/imapplugin.py
@@ -10,6 +10,8 @@ import re
 
 from imap_tools import MailBox, MailMessage
 
+from mariadb import Connection, Cursor
+
 from .imap_pool import MailBoxPool, PoolEmpty
 
 from .db import DBPoolManager
@@ -136,14 +138,17 @@ class ImapPlugin(plugins.SimplePlugin):
 			cur.execute("SELECT tracker,bugid FROM msgindex WHERE messageid=? LIMIT 1",(messageid,))
 			return cur.fetchone()
 	
-	def format_emailaddr(self,project=None,bugid=None,subject=None):
+	def format_emailaddr(self,project=None,bugid=None,subject=None,headers={}):
+		from urllib.parse import quote as quote
 		email=self.addr_format.format(proj=project, bug=bugid)
 		email=email.replace("#None",'')
 		email=email.replace("+None",'')
 		if subject:
-			from urllib.parse import quote as quote
 			email+='?subject='
 			email+=quote(subject, safe='')
+		for key,value in headers.items():
+			email+='?%s=%s'%(key,quote(value,safe=''))
+
 		return email
 
 
@@ -216,7 +221,10 @@ class ImapPlugin(plugins.SimplePlugin):
 					except Exception:
 						import traceback
 						self.mlog("Error processing email: ",traceback.format_exc())
+						self.mlog("Message-ID:",msg.headers["message-id"] if "message-id" in msg.headers else "None")
 						self.mlog("Moving mail to INBOX/Errors.")
+						if "message-id" in msg.headers:
+							self.mail_error(msg=msg,notice="There was an error processing your email with message id:\n "+msg.headers["message-id"]+"\nPlease contact the instance administrator.")
 						self.move_errored_mail(mailbox,msg)
 
 		# block: update all webpages that received new mail
@@ -302,16 +310,50 @@ class ImapPlugin(plugins.SimplePlugin):
 				cur.execute("DELETE FROM subscribers WHERE tracker=? AND bugid=? AND email=?)",(proj,bug,msg.from_))
 			mailbox.delete([msg.uid])
 			return
+		
+		# block: handle secure email which uses $token$ as projectname
+		if proj.startswith('$token$='):
+			token=proj.removeprefix('$token$=')
+			self.mlog("This email is sent to a secure token, resolving (matching against %s)."%token)
+			with self.dbpool.get_connection() as conn, conn.cursor() as cur:
+				cur.execute("SELECT email,tracker,bugid FROM tokens WHERE token=? AND usedin IS NULL",(token,))
+				data=cur.fetchone()
+				if data and data[0]==msg.from_:
+					self.mlog("Resolved to",data)
+					proj=data[1]
+					bug=data[2]
+					try:
+						self.secure_process_commands(mailbox,msg,conn,cur,proj,bug)
+					except Exception as e:
+						self.mlog("Error. IDK.",traceback=True)
+						self.mail_error("Error processing commands:\n "+repr(e)+"\nPlease contact a site admin.")
 
+					cur.execute("UPDATE tokens SET usedin=? WHERE token=?",(msg.headers["message-id"] if "message-id" else "nomessageid"),token)
+					conn.commit()
+				else:
+					self.mlog("Couldn't resolve.")
+		# end block
+
+		# block: handle in-reply-to headers
 		if "in-reply-to" in msg.headers:
 			self.mlog("Using In-Reply-To header to figure out meta")
 			replyid=msg.headers["in-reply-to"]
 			data=self.find_in_reply_to(replyid)
 			if data:
 				proj,bug=data
+
+			elif proj and bug:
+				pass # just ignore this email if we already found a project and bug
+
+			elif "RETRYING" in msg.flags:
+				self.mail_error(msg,"No such email: "+msg.headers["in-reply-to"])
+				self.move_errored_mail(mailbox,msg)
+
 			else: # try again later, maybe we need to index first or handle some other email first
+				mailbox.flag([msg.uid],"RETRYING")
 				self.mlog("Message-ID not in index, trying again next round")
 				return
+		# end block
 
 		# block: make sure a project was specified
 		if not proj:
@@ -379,9 +421,16 @@ class ImapPlugin(plugins.SimplePlugin):
 			if not msg.from_ in bugobj.subscribers:
 				bugobj.addsubscriber(msg.from_)
 
+			if msg.subject.startswith("SECURE"):
+				self.secure_generate_token(mailbox,msg,proj,bug)
+				mailbox.delete([msg.uid])
+				return (proj,bug)
+
 			if not "in-reply-to" in msg.headers:
 				replyto=self.get_newest_from_mailbox(mailbox,proj,bug)
 
+
+			replyto=self.get_newest_from_mailbox(mailbox,proj,bug)
 			self.fwd_to_list(msg,bugobj,replyto=replyto)
 		except Exception:
 			self.mlog("Error processing email '%s' for %s/%d"%(msg.subject,proj,bug),traceback=True)
@@ -434,13 +483,116 @@ class ImapPlugin(plugins.SimplePlugin):
 
 		self.mlog("Bcc:",bcc)
 
+		from_=self.format_emailaddr(bug.tracker,bug.bugid)
+
 		self.smtp.sendmail(
-				from_=self.format_emailaddr(bug.tracker,bug.bugid),
+				from_=from_,
 				bcc=bcc,
 				tosend=msg,
-				replyto=replyto
+				replyto=replyto,
+				extraheaders={"Reply-To":from_}
+				)
+
+	def secure_generate_token(self,mailbox:MailBox,msg:MailMessage,proj:str,bugid:int):
+		self.mlog("Generating a secure token for",msg.from_)
+		import random, string
+		token=''.join(( random.choice(string.ascii_letters) for i in range(40)))
+		with self.dbpool.get_connection() as conn, conn.cursor() as cur:
+			cur.execute("SELECT email FROM permissiontable WHERE email=? AND (tracker=? OR tracker IS NULL) AND (bugid=? OR bugid IS NULL)",(msg.from_,proj,bugid))
+			if not cur.fetchall():
+				self.mail_error(msg,notice="You do not have permission to do that here.")
+				return
+
+			cur.execute("INSERT INTO tokens (email,token,tracker,bugid) VALUES (?,?,?,?)",(msg.from_,token,proj,bugid))
+			conn.commit()
+
+		from_=self.format_emailaddr(project='$token$='+token)
+
+		headers={}
+		self.smtp.format_reply_headers(replyto=self.get_newest_from_mailbox(mailbox,proj,bugid), tosend=headers)
+
+		self.smtp.sendmail(
+				from_=from_,
+				to=msg.from_,
+				subject="Secure token generated for %s/%d"%(proj,bugid),
+				save=False,
+				body="Please reply to this email to use your token, or use the email address:\n "+from_,
+				extraheaders=headers
 				)
 	
+	
+	def secure_process_commands(self,mailbox:MailBox,msg:MailMessage,conn:Connection,cur:Cursor,proj:str,bugid:int):
+		from .emailcommands import findcommands
+		from .types import BUGSTATUS, BUGTYPES
+
+		commandmatches=findcommands(msg.text)
+
+		bug=self.site.getbug(tracker=proj,bugid=bugid)
+
+		for match in commandmatches:
+			text=match.group()
+			text=text.replace("\r\n","\n") # dos2unix so we can forget about \r
+			text=text.lstrip('\n')
+			text=text.rstrip('\n')
+
+			self.mlog(text)
+
+			if text.startswith('SUBJECT '):
+				bug.subject=text.removeprefix("SUBJECT ")
+				self.mlog("Subject changed to:",bug.subject)
+			elif text.startswith("STATUS ") or text.startswith("TYPE "):
+				key,data=text.split(sep=' ',maxsplit=1)
+				possibles={
+						"STATUS":BUGSTATUS,
+						"TYPE":BUGTYPES
+					}
+				if not data in possibles[key]: # no need to errorcheck this, the check above implies key can only be STATUS or TYPE
+					continue
+				
+				self.mlog("Set",key,"to",data)
+				if key=="STATUS":
+					bug.status=data
+				elif key=="TYPE":
+					bug.type=data
+
+			elif text.startswith("DESCRIPTION"):
+				self.mlog("Description command")
+				cmdline,text=text.split('\n',1)
+				text=text.removesuffix("\nDESCRIPTION END")
+				m=re.match(r'DESCRIPTION @([0-9]*)(:[0-9])?',cmdline)
+				start=None
+				end=None
+				self.mlog(m)
+				if m:
+					try:
+						start=int(m[1])
+						if m[1]:
+							end=int(m[2].lstrip(":"))
+						else:
+							end=start+1
+					except ValueError as e:
+						self.mlog("Error converting description range to int:",repr(e))
+						continue
+
+				replacement=text.split("\n")
+				lines=bug.description.split("\n")
+				self.mlog("Replacing: ",repr("\n".join(lines[start:end])),"with",repr("\n".join(replacement)))
+				lines[start:end]=replacement
+				bug.description="\n".join(lines)
+
+			elif text.startswith("PERMIT "):
+				account=text.removeprefix("PERMIT ")
+				cur.execute("INSERT INTO permissiontable VALUES (?,?,?)",(account,proj,bugid))
+				conn.commit()
+			elif text.startswith("UNPERMIT "):
+				account=text.removeprefix("UNPERMIT ")
+				cur.execute("DELETE FROM permissiontable WHERE email=? AND tracker=? AND bugid=?)",(account,proj,bugid))
+				conn.commit()
+
+			else:
+				self.mlog("Not yet implemented command: '%s'"%text)
+
+	
 	def stop(self):
 		"Sets stopping signal, waits for all threads to stop"
 		self.mlog("Stopping. This can take a while.")
diff --git a/src/acit/smtpplugin.py b/src/acit/smtpplugin.py
index 87285b2..8c47f75 100644
--- a/src/acit/smtpplugin.py
+++ b/src/acit/smtpplugin.py
@@ -39,8 +39,10 @@ class SmtpPlugin():
 			cc:list[str]=[],
 			bcc:list[str]=[],
 			replyto:IMAP_Message=None,
+			replytoall:bool=True,
 			tosend:IMAP_Message|Message=None,
 			save=False,
+			extraheaders:dict={}
 			):
 		"""
 		Required arguments:
@@ -59,7 +61,7 @@ class SmtpPlugin():
 		body (str): the email body
 		cc (list): email addresses to put in the cc
 
-		save (bool): force adding from_ to the cclist, making ImapPlugin store the email in ``Sent`` on arrival
+		save (bool): add from_ to the cclist, making ImapPlugin store the email in ``Sent`` on arrival
 		"""
 
 		if type(to)==str:
@@ -83,27 +85,31 @@ class SmtpPlugin():
 				else:
 					tosend["cc"]=from_
 
-			tosend["X-Acit-Delete-When-Sender"]=from_
+			if not save:
+				tosend["X-Acit-Delete-When-Sender"]=from_
 
 		else:
 
 			if replyto:
-				cc.extend(replyto.cc)
-				cc.extend(replyto.to)
-				if not save:
-					cc.remove(from_)
+				if replytoall:
+					cc.extend(replyto.cc)
+					cc.extend(replyto.to)
+
+				if from_ in cc: cc.remove(from_)
+				if from_ in bcc: bcc.remove(from_)
+
 				to.append(replyto.headers["reply-to"] if "reply-to" in replyto.headers else replyto.from_)
 
 				if not subject:
 					subject="Re: "+replyto.subject
 
 
-			for addr in to:
-				if addr in cc:
-					cc.remove(addr)
 			for addr in cc:
 				if addr in bcc:
 					bcc.remove(addr)
+			for addr in to:
+				if addr in cc:
+					cc.remove(addr)
 
 			if not (to and body and from_ and subject):
 				raise ValueError("Missing to, from_, body, or subject")
@@ -112,22 +118,11 @@ class SmtpPlugin():
 			tosend["From"]=from_
 			tosend["To"]=", ".join(set(to)) # set()s can't have duplicates
 			tosend["Subject"]=subject
-			tosend["Cc"]=", ".join(set(cc))
-			tosend["Bcc"]=", ".join(set(bcc))
+			if cc: tosend["Cc"]=", ".join(set(cc))
+			if bcc: tosend["Bcc"]=", ".join(set(bcc))
 
 			if replyto:
-				if "references" in replyto.headers:
-					tosend["References"]=replyto.obj["references"]
-				elif "in-reply-to" in replyto.headers:
-					tosend["References"]=replyto.obj["in-reply-to"]
-
-				if "message-id" in replyto.headers:
-					tosend["In-Reply-To"]=replyto.obj["message-id"]
-
-					if "References" in tosend:
-						tosend["References"]+=" "+replyto.obj["message-id"]
-					else:
-						tosend["References"]=replyto.obj["message-id"]
+				self.format_reply_headers(replyto=replyto,tosend=tosend) # tosend is mutable so we can ignore the return value
 
 			tosend.set_payload(body)
 
@@ -140,9 +135,29 @@ class SmtpPlugin():
 		if save and not from_ in tosend["Cc"]:
 			tosend["Cc"]+=" "+from_
 
+		for key,value in extraheaders.items():
+			tosend[key]=value
+
 		self.que.put(tosend)
 
 		return tosend["Message-ID"]
+	
+
+	def format_reply_headers(self,replyto:IMAP_Message,tosend:Message|dict={}):
+		if "references" in replyto.headers:
+			tosend["References"]=replyto.obj["references"]
+		elif "in-reply-to" in replyto.headers:
+			tosend["References"]=replyto.obj["in-reply-to"]
+
+		if "message-id" in replyto.headers:
+			tosend["In-Reply-To"]=replyto.obj["message-id"]
+
+			if "References" in tosend:
+				tosend["References"]+=" "+replyto.obj["message-id"]
+			else:
+				tosend["References"]=replyto.obj["message-id"]
+
+		return tosend
 
 	
 	def start(self):
@@ -191,6 +206,7 @@ class SmtpPlugin():
 				import traceback
 				self.mlog("An error occured in the SMTP runner:\n",traceback.format_exc())
 				self.mlog("As a result, the following email was discarded: %s '%s'"%(item.get("message-id"),item.get("subject")))
+				self.mlog("Raw:\n"+item.as_string())
 
 			finally:
 				self.que.task_done()
diff --git a/src/acit/types.py b/src/acit/types.py
index fd9d20a..3d41297 100644
--- a/src/acit/types.py
+++ b/src/acit/types.py
@@ -47,11 +47,21 @@ class Site():
 				")"
 				)
 
-			#cur.execute(
-			#	"CREATE TABLE IF NOT EXISTS accounts ("
-			#	"email VARCHAR(80),"
-			#	"hash VARCHAR(80),"
-			# 	)# TODO
+			cur.execute(
+				"CREATE TABLE IF NOT EXISTS permissiontable ("
+				"email VARCHAR(80),"
+				"tracker VARCHAR(80),"
+				"bugid INT)"
+			 	)
+
+			cur.execute(
+				"CREATE TABLE IF NOT EXISTS tokens ("
+				"email VARCHAR(80),"
+				"token VARCHAR(80),"
+				"tracker VARCHAR(80),"
+				"bugid INT,"
+				"usedin TINYTEXT DEFAULT NULL)"
+			 	)
 			
 			conn.commit()
 
@@ -104,12 +114,13 @@ class Site():
 
 							name,homepage=data
 
-							log(name)
+							#log(name)
 
 							cur.execute("REPLACE INTO trackers (name,homepage) VALUES (?,?)",(name,homepage))
 							conn.commit()
 
-							self.update_all_bugpages(conn,cur,trackerfilter=name)
+							#self.update_all_bugpages(conn,cur,trackerfilter=name)
+							cherrypy.engine.publish("regen",name,None)
 
 					self.last_tracker_update=datetime.now()
 
@@ -346,7 +357,7 @@ class Tracker():
 		NOTE: this executes an SQL query.
 		"""
 		with self.site.dbpool.get_connection() as conn, conn.cursor() as cur:
-			cur.execute("SELECT email FROM subscribers WHERE tracker=? AND bugid=NULL",(self.tracker,))
+			cur.execute("SELECT email FROM subscribers WHERE tracker=? AND bugid IS NULL",(self.tracker,))
 			return [ value[0] for value in cur ]
 	
 	def addsubscriber(self,email):
@@ -356,6 +367,6 @@ class Tracker():
 	
 	def rmsubscriber(self,email):
 		with self.site.dbpool.get_connection() as conn, conn.cursor() as cur:
-			cur.execute("DELETE FROM subscribers WHERE tracker=? AND bugid=NULL AND email=?)",(self.tracker,email))
+			cur.execute("DELETE FROM subscribers WHERE tracker=? AND bugid IS NULL AND email=?)",(self.tracker,email))
 			conn.commit()