A lot of changes

- don't send emails multiple times
- fix the forwarding procedure
- send secure tokens automatically instead of on-request
- ignore duplicate emails (matches by message-id)
- allow passing an SMTP `to` to `sendmail()`, instead of always using
  the headers
- don't add self to cc when not needed, etc etc
- introduce a postprocessor function mechanism that runs a function
  right after our sendmail() processes your email, and right before
  adding it to the smtp queue

3 files changed, 164 insertions, 33 deletions

diff --git a/src/acit/imapplugin.py b/src/acit/imapplugin.py
index 2ceb0a1..6a85855 100644
--- a/src/acit/imapplugin.py
+++ b/src/acit/imapplugin.py
@@ -8,7 +8,7 @@ import cherrypy
 
 import re
 
-from imap_tools import MailBox, MailMessage
+from imap_tools import MailBox, MailMessage, MailMessageFlags
 
 from mariadb import Connection, Cursor
 
@@ -78,6 +78,8 @@ class ImapPlugin(plugins.SimplePlugin):
 
 		self.smtp.domain=self.emaildomain
 
+		self.msgids_sorted_not_indexed=[]
+
 		self.mlog("Starting mailbox pool. This can take a while.")
 		self.mbpool.open()
 		pool=self.mbpool.get_pool_size()
@@ -125,7 +127,10 @@ class ImapPlugin(plugins.SimplePlugin):
 					mb.folder.set(self.get_bug_folder(mb,tracker,bugid))
 					for msg in mb.fetch():
 						if "message-id" in msg.headers:
-							cur.execute("REPLACE INTO msgindex VALUES (?,?,?)",(tracker,bugid,msg.headers["message-id"][:255]))
+							msgid=msg.headers["message-id"]
+							if msgid in self.msgids_sorted_not_indexed:
+								self.msgids_sorted_not_indexed.remove(msgid)
+							cur.execute("REPLACE INTO msgindex VALUES (?,?,?)",(tracker,bugid,msgid[:255]))
 
 				except Exception as e:
 					self.mlog("Error while indexing mailbox of %s/%d: %s"%(tracker,bugid,e))
@@ -141,6 +146,7 @@ class ImapPlugin(plugins.SimplePlugin):
 	def format_emailaddr(self,project=None,bugid=None,subject=None,headers={}):
 		from urllib.parse import quote as quote
 		email=self.addr_format.format(proj=project, bug=bugid)
+		email=email.replace("None#","#")
 		email=email.replace("#None",'')
 		email=email.replace("+None",'')
 		if subject:
@@ -214,6 +220,8 @@ class ImapPlugin(plugins.SimplePlugin):
 					try:
 						target=self.handle_email(mailbox,msg)
 
+						if "message-id" in msg.headers: self.msgids_sorted_not_indexed.append(msg.headers["message-id"])
+
 						if target:
 							proj,bug=target
 							refreshable.setdefault(proj,[]).append(bug)
@@ -224,7 +232,7 @@ class ImapPlugin(plugins.SimplePlugin):
 						self.mlog("Message-ID:",msg.headers["message-id"] if "message-id" in msg.headers else "None")
 						self.mlog("Moving mail to INBOX/Errors.")
 						if "message-id" in msg.headers:
-							self.mail_error(msg=msg,notice="There was an error processing your email with message id:\n "+msg.headers["message-id"]+"\nPlease contact the instance administrator.")
+							self.mail_error(msg=msg,notice="There was an error processing your email.\nPlease contact the instance administrator.")
 						self.move_errored_mail(mailbox,msg)
 
 		# block: update all webpages that received new mail
@@ -250,8 +258,25 @@ class ImapPlugin(plugins.SimplePlugin):
 	
 
 	def handle_email(self,mailbox:MailBox,msg:MailMessage):
+		if "message-id" in msg.headers:
+			msgid=msg.headers["message-id"]
+			skip=False
+			if msgid in self.msgids_sorted_not_indexed:
+				skip=True
+			else:
+				with self.dbpool.get_connection() as conn, conn.cursor() as cur:
+					cur.execute("SELECT messageid FROM msgindex WHERE messageid=? LIMIT 1",(msgid))
+					if cur.fetchone():
+						skip=True
+			if skip:
+				self.mlog("An email with messageid %s already exists, skipping"%msgid)
+				mailbox.flag([msg.uid],MailMessageFlags.SEEN,value=False) # mark as unread
+				mailbox.move([msg.uid],self.ensurefolder(mailbox,"INBOX","duplicates"))
+				return
+
 		self.mlog("Processing email with subject '%s'"%msg.subject)
 
+		
 		if "x-acit-delete-when-sender" in msg.headers and "sender" in msg.headers:
 			if msg.headers["x-acit-delete-when-sender"]==msg.headers["sender"]:
 				self.mlog("Header requested deletion, deleting email")
@@ -265,13 +290,21 @@ class ImapPlugin(plugins.SimplePlugin):
 				return
 
 
+		proj=None
+		bug=None
 		for addr in msg.to + msg.cc + msg.bcc + msg.reply_to:
 			if re.fullmatch(self.addr_regex,addr):
-				proj,bug=self.stripInfoFromMailAddr(addr)
-				break
-		else:
-			proj=None
-			bug=None
+				newproj,newbug=self.stripInfoFromMailAddr(addr)
+				if newproj and not proj or proj==newproj: # if we can set proj, set it.
+					proj=newproj
+					if newbug and not bug: # if we can set a bugid, set it too.
+						# the reason this block is below the project block, is to ensure we don't set a bugid with an unrelated proj
+						bug=newbug
+
+				if newproj.startswith("$token$="):
+					proj=newproj
+					break # ensure no further things get set, we wanna use the token
+
 
 		if msg.subject.startswith("SUBSCRIBE"):
 			from_=self.format_emailaddr(project=proj,bugid=bug)
@@ -312,13 +345,13 @@ class ImapPlugin(plugins.SimplePlugin):
 			return
 		
 		# block: handle secure email which uses $token$ as projectname
-		if proj.startswith('$token$='):
+		if proj and proj.startswith('$token$='):
 			token=proj.removeprefix('$token$=')
 			self.mlog("This email is sent to a secure token, resolving (matching against %s)."%token)
 			with self.dbpool.get_connection() as conn, conn.cursor() as cur:
-				cur.execute("SELECT email,tracker,bugid FROM tokens WHERE token=? AND usedin IS NULL",(token,))
+				cur.execute("SELECT email,tracker,bugid,usedin FROM tokens WHERE token=? AND usedin IS NULL",(token,))
 				data=cur.fetchone()
-				if data and data[0]==msg.from_:
+				if data and data[0]==msg.from_: # sender matches, or token was used before
 					self.mlog("Resolved to",data)
 					proj=data[1]
 					bug=data[2]
@@ -328,6 +361,7 @@ class ImapPlugin(plugins.SimplePlugin):
 						self.mlog("Error. IDK.",traceback=True)
 						self.mail_error("Error processing commands:\n "+repr(e)+"\nPlease contact a site admin.")
 
+					# set token used state
 					cur.execute("UPDATE tokens SET usedin=? WHERE token=?",(msg.headers["message-id"] if "message-id" else "nomessageid"),token)
 					conn.commit()
 				else:
@@ -477,20 +511,97 @@ class ImapPlugin(plugins.SimplePlugin):
 		return mailbox.move(msg.uid,target)
 	
 
-	def fwd_to_list(self,msg:MailMessage,bug:Bug,replyto:MailMessage=None):
-		tracker=self.site.gettracker(bug.tracker)
-		bcc=bug.subscribers + tracker.subscribers
+	def email_postprocessor_fix_acit_a_thousand_times(self,email,realacitfrom=None):
+		if not realacitfrom:
+			realacitfrom=self.format_emailaddr()
+
+		acit_re_added=False
+		for header in "to", "cc":
+			if not header in email:
+				continue
+			cur=email[header]
+			new=""
+			while True:
+				match=re.search(self.addr_regex+"(, )?",cur)
+				if match:
+					self.mlog("Removing",match.group(),"from",header)
+					new+=match.string[:match.start()]
+					cur=match.string[match.end():]
+					if not acit_re_added:
+						self.mlog("Readding",realacitfrom,"instead")
+						new+=realacitfrom
+						acit_re_added=True
+					self.mlog(header,"= ",new,"<<",cur)
+				else:
+					break
+
+			if new:
+				del email[header]
+				email[header]=new
+
+		if not ( realacitfrom in email["to"] or realacitfrom in email.get("cc","") ):
+			cc=( email["cc"]+", " if email["cc"] else "" )
+			del email["cc"]
+			email["Cc"]=cc+realacitfrom
+			
+
+	def fwd_to_list(self,msg:MailMessage,bug:Bug,replyto:MailMessage=None,bcc:list=None):
+		import random, string
+		if not bcc:
+			tracker=self.site.gettracker(bug.tracker)
+			bcc=bug.subscribers + tracker.subscribers
 
 		self.mlog("Bcc:",bcc)
 
+		securereplytos={}
+		with self.dbpool.get_connection() as conn, conn.cursor() as cur:
+			for email in bcc:
+				try:
+					cur.execute("SELECT email FROM permissiontable WHERE email=? AND (tracker=? OR tracker IS NULL) AND (bugid=? OR bugid IS NULL) LIMIT 1",(email,bug.tracker,bug.bugid))
+					if cur.fetchone():
+						token=''.join(( random.choice(string.ascii_letters) for i in range(40)))
+						cur.execute("UPDATE tokens SET usedin='@@UNUSED@@' WHERE email=? AND tracker=? AND bugid=? AND USEDIN IS NULL",(email,bug.tracker,bug.bugid))
+
+						cur.execute("INSERT INTO tokens (email,token,tracker,bugid) VALUES (?,?,?,?)",(msg.from_,token,bug.tracker,bug.bugid))
+						securereplytos[email]=token
+						self.mlog("Token",token,"generated for",email)
+
+				except Exception:
+					self.mlog("Error generating a secure email token for",email,traceback=True)
+
+			conn.commit()
+
+		for email in securereplytos.keys():
+			self.mlog("Removing",email,"from bcc")
+			while email in bcc:
+				bcc.remove(email)
+
+		self.mlog("Bcc 2:",bcc)
+
 		from_=self.format_emailaddr(bug.tracker,bug.bugid)
 
+		kwargs={
+			"tosend":msg,
+			"replyto":replyto,
+			}
+
 		self.smtp.sendmail(
+			from_=from_,
+			bcc=bcc,
+			postprocessor=lambda msg: self.email_postprocessor_fix_acit_a_thousand_times(msg, from_),
+			smtp_to=bcc,
+			**kwargs
+			)
+
+		for email,token in securereplytos.items():
+			self.mlog("Sending extra email to %s bc of token"%email)
+			from_=self.format_emailaddr(project='$token$='+token)
+			self.smtp.sendmail(
+				to=email,
 				from_=from_,
-				bcc=bcc,
-				tosend=msg,
-				replyto=replyto,
-				extraheaders={"Reply-To":from_}
+				postprocessor=lambda msg: self.email_postprocessor_fix_acit_a_thousand_times(msg, from_),
+				smtp_to=email,
+				**kwargs
 				)
 
 	def secure_generate_token(self,mailbox:MailBox,msg:MailMessage,proj:str,bugid:int):
diff --git a/src/acit/smtpplugin.py b/src/acit/smtpplugin.py
index fafa892..0abb9eb 100644
--- a/src/acit/smtpplugin.py
+++ b/src/acit/smtpplugin.py
@@ -1,4 +1,6 @@
 
+from typing import Callable
+
 import cherrypy
 import threading
 from os import getenv
@@ -42,7 +44,10 @@ class SmtpPlugin():
 			replytoall:bool=True,
 			tosend:IMAP_Message|Message=None,
 			save=False,
-			extraheaders:dict={}
+			extraheaders:dict={},
+			overwriteheaders:dict={},
+			postprocessor:Callable=None,
+			smtp_to:list|str=None,
 			):
 		"""
 		Required arguments:
@@ -74,17 +79,12 @@ class SmtpPlugin():
 			tosend["Resent-From"]=from_
 			tosend["Resent-Date"]=utils.format_datetime(datetime.now())
 			tosend["Resent-To"]=", ".join(to)
+			tosend["Resent-Message-ID"]=self.gen_msg_id()
 			if bcc:
 				tosend["Resent-Bcc"]=", ".join(bcc)
 			if cc:
 				tosend["Resent-Cc"]=", ".join(bcc)
 
-			if not from_ in tosend["to"]+tosend.get("cc","")+tosend.get("resent-to",""):
-				if "cc" in tosend:
-					tosend["cc"]+=", "+from_
-				else:
-					tosend["cc"]=from_
-
 			if not save:
 				tosend["X-Acit-Delete-When-Sender"]=from_
 
@@ -129,8 +129,8 @@ class SmtpPlugin():
 
 		tosend["Sender"]=from_
 		tosend["X-Acit-Is-Outgoing"]=from_
-		if not "Message-ID" in tosend:
-			tosend["Message-ID"]="<"+datetime.now().strftime("%d%m%Y%H%M%S%j")+"@"+self.domain+">"
+		if not "message-id" in tosend:
+			tosend["Message-ID"]=self.gen_msg_id()
 
 		if save and not from_ in tosend["Cc"]:
 			tosend["Cc"]+=" "+from_
@@ -138,11 +138,25 @@ class SmtpPlugin():
 		for key,value in extraheaders.items():
 			tosend[key]=value
 
-		self.que.put(tosend)
+		for key,value in overwriteheaders.items():
+			del tosend[key]
+			tosend[key]=value
+
+		if callable(postprocessor):
+			postprocessor(tosend)
+
+		self.que.put({"msg":tosend,"real-to":smtp_to})
 
-		return tosend["Message-ID"]
+		import traceback
+		function=traceback.extract_stack(limit=2)[0].name
+		self.mlog("sendmail called by",function,"for",tosend["message-id"])
+
+		return tosend["message-id"]
 	
 
+	def gen_msg_id(self):
+		return "<"+datetime.now().strftime("%d%m%Y%H%M%S%j")+"@"+self.domain+">"
+
 	def format_reply_headers(self,replyto:IMAP_Message,tosend:Message|dict={}):
 		if "references" in replyto.headers:
 			tosend["References"]=replyto.obj["references"]
@@ -169,8 +183,8 @@ class SmtpPlugin():
 		smtp=None
 		while True:
 			try:
-				item=self.que.get(timeout=1)
-				if not item:
+				data=self.que.get(timeout=1)
+				if not data:
 					continue
 			except Empty:
 				if smtp:
@@ -181,6 +195,7 @@ class SmtpPlugin():
 				continue
 
 			try:
+				item=data["msg"]
 				if not smtp:
 					self.mlog("Connecting and logging in to smtp server")
 					smtp=SMTP(host=self.smtp_server,port=self.smtp_port)
@@ -189,7 +204,7 @@ class SmtpPlugin():
 
 				self.mlog("Sending message %s '%s'"%(item.get("message-id"),item.get("subject")))
 
-				smtp.send_message(item)
+				smtp.send_message(item, to_addrs=data["real-to"])
 
 			except SMTPAuthenticationError:
 				self.mlog("!!! AUTH ERROR !!! please check your config. Discarding email.")
diff --git a/src/acit/types.py b/src/acit/types.py
index 3d41297..37f1e8c 100644
--- a/src/acit/types.py
+++ b/src/acit/types.py
@@ -99,6 +99,9 @@ class Site():
 					script=getenv("ACIT_LIST_TRACKERS","/usr/lib/acit-list-trackers")
 					proc=subprocess.run(script,capture_output=True)
 
+					cur.execute("SELECT name FROM trackers")
+					old_trackers=[ item[0] for item in cur ]
+
 					if proc.stderr:
 						log("Refresh script generated STDERR:")
 						log(proc.stderr)
@@ -116,11 +119,13 @@ class Site():
 
 							#log(name)
 
+
 							cur.execute("REPLACE INTO trackers (name,homepage) VALUES (?,?)",(name,homepage))
 							conn.commit()
 
 							#self.update_all_bugpages(conn,cur,trackerfilter=name)
-							cherrypy.engine.publish("regen",name,None)
+							if not name in old_trackers:
+								cherrypy.engine.publish("regen",name,None)
 
 					self.last_tracker_update=datetime.now()