diff options
Diffstat (limited to 'src/acit/imapplugin.py')
| -rw-r--r-- | src/acit/imapplugin.py | 147 |
1 files changed, 129 insertions, 18 deletions
diff --git a/src/acit/imapplugin.py b/src/acit/imapplugin.py index 2ceb0a1..6a85855 100644 --- a/src/acit/imapplugin.py +++ b/src/acit/imapplugin.py @@ -8,7 +8,7 @@ import cherrypy import re -from imap_tools import MailBox, MailMessage +from imap_tools import MailBox, MailMessage, MailMessageFlags from mariadb import Connection, Cursor @@ -78,6 +78,8 @@ class ImapPlugin(plugins.SimplePlugin): self.smtp.domain=self.emaildomain + self.msgids_sorted_not_indexed=[] + self.mlog("Starting mailbox pool. This can take a while.") self.mbpool.open() pool=self.mbpool.get_pool_size() @@ -125,7 +127,10 @@ class ImapPlugin(plugins.SimplePlugin): mb.folder.set(self.get_bug_folder(mb,tracker,bugid)) for msg in mb.fetch(): if "message-id" in msg.headers: - cur.execute("REPLACE INTO msgindex VALUES (?,?,?)",(tracker,bugid,msg.headers["message-id"][:255])) + msgid=msg.headers["message-id"] + if msgid in self.msgids_sorted_not_indexed: + self.msgids_sorted_not_indexed.remove(msgid) + cur.execute("REPLACE INTO msgindex VALUES (?,?,?)",(tracker,bugid,msgid[:255])) except Exception as e: self.mlog("Error while indexing mailbox of %s/%d: %s"%(tracker,bugid,e)) @@ -141,6 +146,7 @@ class ImapPlugin(plugins.SimplePlugin): def format_emailaddr(self,project=None,bugid=None,subject=None,headers={}): from urllib.parse import quote as quote email=self.addr_format.format(proj=project, bug=bugid) + email=email.replace("None#","#") email=email.replace("#None",'') email=email.replace("+None",'') if subject: @@ -214,6 +220,8 @@ class ImapPlugin(plugins.SimplePlugin): try: target=self.handle_email(mailbox,msg) + if "message-id" in msg.headers: self.msgids_sorted_not_indexed.append(msg.headers["message-id"]) + if target: proj,bug=target refreshable.setdefault(proj,[]).append(bug) @@ -224,7 +232,7 @@ class ImapPlugin(plugins.SimplePlugin): self.mlog("Message-ID:",msg.headers["message-id"] if "message-id" in msg.headers else "None") self.mlog("Moving mail to INBOX/Errors.") if "message-id" in msg.headers: - self.mail_error(msg=msg,notice="There was an error processing your email with message id:\n "+msg.headers["message-id"]+"\nPlease contact the instance administrator.") + self.mail_error(msg=msg,notice="There was an error processing your email.\nPlease contact the instance administrator.") self.move_errored_mail(mailbox,msg) # block: update all webpages that received new mail @@ -250,8 +258,25 @@ class ImapPlugin(plugins.SimplePlugin): def handle_email(self,mailbox:MailBox,msg:MailMessage): + if "message-id" in msg.headers: + msgid=msg.headers["message-id"] + skip=False + if msgid in self.msgids_sorted_not_indexed: + skip=True + else: + with self.dbpool.get_connection() as conn, conn.cursor() as cur: + cur.execute("SELECT messageid FROM msgindex WHERE messageid=? LIMIT 1",(msgid)) + if cur.fetchone(): + skip=True + if skip: + self.mlog("An email with messageid %s already exists, skipping"%msgid) + mailbox.flag([msg.uid],MailMessageFlags.SEEN,value=False) # mark as unread + mailbox.move([msg.uid],self.ensurefolder(mailbox,"INBOX","duplicates")) + return + self.mlog("Processing email with subject '%s'"%msg.subject) + if "x-acit-delete-when-sender" in msg.headers and "sender" in msg.headers: if msg.headers["x-acit-delete-when-sender"]==msg.headers["sender"]: self.mlog("Header requested deletion, deleting email") @@ -265,13 +290,21 @@ class ImapPlugin(plugins.SimplePlugin): return + proj=None + bug=None for addr in msg.to + msg.cc + msg.bcc + msg.reply_to: if re.fullmatch(self.addr_regex,addr): - proj,bug=self.stripInfoFromMailAddr(addr) - break - else: - proj=None - bug=None + newproj,newbug=self.stripInfoFromMailAddr(addr) + if newproj and not proj or proj==newproj: # if we can set proj, set it. + proj=newproj + if newbug and not bug: # if we can set a bugid, set it too. + # the reason this block is below the project block, is to ensure we don't set a bugid with an unrelated proj + bug=newbug + + if newproj.startswith("$token$="): + proj=newproj + break # ensure no further things get set, we wanna use the token + if msg.subject.startswith("SUBSCRIBE"): from_=self.format_emailaddr(project=proj,bugid=bug) @@ -312,13 +345,13 @@ class ImapPlugin(plugins.SimplePlugin): return # block: handle secure email which uses $token$ as projectname - if proj.startswith('$token$='): + if proj and proj.startswith('$token$='): token=proj.removeprefix('$token$=') self.mlog("This email is sent to a secure token, resolving (matching against %s)."%token) with self.dbpool.get_connection() as conn, conn.cursor() as cur: - cur.execute("SELECT email,tracker,bugid FROM tokens WHERE token=? AND usedin IS NULL",(token,)) + cur.execute("SELECT email,tracker,bugid,usedin FROM tokens WHERE token=? AND usedin IS NULL",(token,)) data=cur.fetchone() - if data and data[0]==msg.from_: + if data and data[0]==msg.from_: # sender matches, or token was used before self.mlog("Resolved to",data) proj=data[1] bug=data[2] @@ -328,6 +361,7 @@ class ImapPlugin(plugins.SimplePlugin): self.mlog("Error. IDK.",traceback=True) self.mail_error("Error processing commands:\n "+repr(e)+"\nPlease contact a site admin.") + # set token used state cur.execute("UPDATE tokens SET usedin=? WHERE token=?",(msg.headers["message-id"] if "message-id" else "nomessageid"),token) conn.commit() else: @@ -477,20 +511,97 @@ class ImapPlugin(plugins.SimplePlugin): return mailbox.move(msg.uid,target) - def fwd_to_list(self,msg:MailMessage,bug:Bug,replyto:MailMessage=None): - tracker=self.site.gettracker(bug.tracker) - bcc=bug.subscribers + tracker.subscribers + def email_postprocessor_fix_acit_a_thousand_times(self,email,realacitfrom=None): + if not realacitfrom: + realacitfrom=self.format_emailaddr() + + acit_re_added=False + for header in "to", "cc": + if not header in email: + continue + cur=email[header] + new="" + while True: + match=re.search(self.addr_regex+"(, )?",cur) + if match: + self.mlog("Removing",match.group(),"from",header) + new+=match.string[:match.start()] + cur=match.string[match.end():] + if not acit_re_added: + self.mlog("Readding",realacitfrom,"instead") + new+=realacitfrom + acit_re_added=True + self.mlog(header,"= ",new,"<<",cur) + else: + break + + if new: + del email[header] + email[header]=new + + if not ( realacitfrom in email["to"] or realacitfrom in email.get("cc","") ): + cc=( email["cc"]+", " if email["cc"] else "" ) + del email["cc"] + email["Cc"]=cc+realacitfrom + + + def fwd_to_list(self,msg:MailMessage,bug:Bug,replyto:MailMessage=None,bcc:list=None): + import random, string + if not bcc: + tracker=self.site.gettracker(bug.tracker) + bcc=bug.subscribers + tracker.subscribers self.mlog("Bcc:",bcc) + securereplytos={} + with self.dbpool.get_connection() as conn, conn.cursor() as cur: + for email in bcc: + try: + cur.execute("SELECT email FROM permissiontable WHERE email=? AND (tracker=? OR tracker IS NULL) AND (bugid=? OR bugid IS NULL) LIMIT 1",(email,bug.tracker,bug.bugid)) + if cur.fetchone(): + token=''.join(( random.choice(string.ascii_letters) for i in range(40))) + cur.execute("UPDATE tokens SET usedin='@@UNUSED@@' WHERE email=? AND tracker=? AND bugid=? AND USEDIN IS NULL",(email,bug.tracker,bug.bugid)) + + cur.execute("INSERT INTO tokens (email,token,tracker,bugid) VALUES (?,?,?,?)",(msg.from_,token,bug.tracker,bug.bugid)) + securereplytos[email]=token + self.mlog("Token",token,"generated for",email) + + except Exception: + self.mlog("Error generating a secure email token for",email,traceback=True) + + conn.commit() + + for email in securereplytos.keys(): + self.mlog("Removing",email,"from bcc") + while email in bcc: + bcc.remove(email) + + self.mlog("Bcc 2:",bcc) + from_=self.format_emailaddr(bug.tracker,bug.bugid) + kwargs={ + "tosend":msg, + "replyto":replyto, + } + self.smtp.sendmail( + from_=from_, + bcc=bcc, + postprocessor=lambda msg: self.email_postprocessor_fix_acit_a_thousand_times(msg, from_), + smtp_to=bcc, + **kwargs + ) + + for email,token in securereplytos.items(): + self.mlog("Sending extra email to %s bc of token"%email) + from_=self.format_emailaddr(project='$token$='+token) + self.smtp.sendmail( + to=email, from_=from_, - bcc=bcc, - tosend=msg, - replyto=replyto, - extraheaders={"Reply-To":from_} + postprocessor=lambda msg: self.email_postprocessor_fix_acit_a_thousand_times(msg, from_), + smtp_to=email, + **kwargs ) def secure_generate_token(self,mailbox:MailBox,msg:MailMessage,proj:str,bugid:int): |