#!/usr/bin/env python3.4
# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
 #the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" Publish notifications about mails to pony mail.

Copy this file to $mailman_plugin_dir/mailman_ponymail/__init__.py
Also copy ponymail.cfg to that dir.
Enable the module by adding the following to your mailman.cfg file::

[archiver.ponymail]
# The class implementing the IArchiver interface.
class: mailman_ponymail_plugin.Archiver
enable: yes

OR, to use the STDIN version (non-MM3 mailing list managers),
sub someone to the list(s) and add this to their .forward file:
"|/usr/bin/env python3.4 /path/to/archiver.py"

"""

# Change this index name to whatever you picked!!
indexname = "ponymail_alpha"
logger = None
if __name__ != '__main__':
    from zope.interface import implementer
    from mailman.interfaces.archiver import IArchiver
    from mailman.interfaces.archiver import ArchivePolicy
    import logging
    logger = logging.getLogger("mailman.archiver")
else:
    import sys
    import argparse

from elasticsearch import Elasticsearch
import hashlib
import email.utils
import datetime, time
import json
from collections import namedtuple
import re
import codecs
import configparser
import os
import fnmatch

# Fetch config
path = os.path.dirname(os.path.realpath(__file__))
config = configparser.RawConfigParser()
config.read("%s/ponymail.cfg" % path)
auth = None
if config.has_option('elasticsearch', 'user'):
    auth = (config.get('elasticsearch','user'), config.get('elasticsearch','password'))

def parse_attachment(part):
    cd = part.get("Content-Disposition", None)
    if cd:
        dispositions = cd.strip().split(";")
        if dispositions[0].lower() == "attachment":
            fd = part.get_payload(decode=True)
            attachment = {}
            attachment['content_type'] = part.get_content_type()
            attachment['size'] = len(fd)
            attachment['filename'] = None
            h = hashlib.sha256(fd).hexdigest()
            b64 = codecs.encode(fd, "base64").decode('ascii', errors='ignore')
            attachment['hash'] = h
            for param in dispositions[1:]:
                key,val = param.split("=")
                if key.lower().strip() == "filename":
                    val = val.strip(' "')
                    print("Found attachment: %s" % val)
                    attachment['filename'] = val
            if attachment['filename']:
                return attachment, b64 # Return meta data and contents separately
    return None, None

def pm_charsets(msg):
    charsets = set({})
    for c in msg.get_charsets():
        if c is not None:
            charsets.update([c])
    return charsets

class Archiver(object):
    """ A mailman 3 archiver that forwards messages to pony mail. """
    if __name__ != '__main__':
        implementer(IArchiver)
    name = "ponymail"

    # This is a list of the headers we're interested in publishing.
    keys = [
        "archived-at",
        "delivered-to",
        "from",
        "cc",
        "to",
        "date",
        "in-reply-to",
        "message-id",
        "subject",
        "x-message-id-hash",
        "references",
        "x-mailman-rule-hits",
        "x-mailman-rule-misses",
    ]

    def __init__(self):
        """ Just initialize ES. """
        global config, auth
        ssl = False
        self.cropout = None
        self.dbname = config.get("elasticsearch", "dbname")
        if config.has_option("elasticsearch", "ssl") and config.get("elasticsearch", "ssl").lower() == 'true':
            ssl = True
        if config.has_option("debug", "cropout") and config.get("debug", "cropout") != "":
            self.cropout = config.get("debug", "cropout")
        uri = ""
        if config.has_option("elasticsearch", "uri") and config.get("elasticsearch", "uri") != "":
            uri = config.get("elasticsearch", "uri")
        self.es = Elasticsearch([
            {
                'host': config.get("elasticsearch", "hostname"),
                'port': int(config.get("elasticsearch", "port")),
                'use_ssl': ssl,
                'url_prefix': uri,
                'http_auth': auth
            }],
            max_retries=5,
            retry_on_timeout=True
            )

    def msgfiles(self, msg):
        attachments = []
        contents = {}
        if msg.is_multipart():    
            for part in msg.walk():
                part_meta, part_file = parse_attachment(part)
                if part_meta:
                    attachments.append(part_meta)
                    contents[part_meta['hash']] = part_file
        return attachments, contents
    
    
    def msgbody(self, msg):
        body = None
        if msg.is_multipart():    
            for part in msg.walk():
                if part.is_multipart(): 
                    for subpart in part.walk():
                        if subpart.get_content_type() == 'text/plain':
                                body = subpart.get_payload(decode=True)
                                break
        
                elif part.get_content_type() == 'text/plain':
                    body = part.get_payload(decode=True)
                    break
        
        elif msg.get_content_type() == 'text/plain':
            body = msg.get_payload(decode=True) 
    
        for charset in pm_charsets(msg):
            try:
                body = body.decode(charset) if type(body) is bytes else body
            except:
                body = body.decode('utf-8', errors='replace') if type(body) is bytes else body
                
        return body   

    def archive_message(self, mlist, msg):
        """Send the message to the archiver.

        :param mlist: The IMailingList object.
        :param msg: The message object.
        """

        lid = None
        m = re.search(r"(<.+>)", mlist.list_id.replace("@", "."))
        if m:
            lid = m.group(1)
        else:
            lid = "<%s>" % mlist.list_id.strip("<>").replace("@", ".")
        if self.cropout:
            lid = lid.replace(self.cropout, "")
        
        format = lambda value: value and str(value) or ""
        msg_metadata = dict([(k, format(msg.get(k))) for k in self.keys])
        mid = hashlib.sha224(str("%s-%s" % (lid, msg_metadata['archived-at'])).encode('utf-8')).hexdigest() + "@" + (lid if lid else "none")
        for key in ['to','from','subject','message-id']:
            try:
                hval = ""
                if msg_metadata.get(key):
                    for t in email.header.decode_header(msg_metadata[key]):
                        if t[1] == None or t[1].find("8bit") != -1:
                            hval += t[0].decode('utf-8') if type(t[0]) is bytes else t[0]
                        else:
                            hval += t[0].decode(t[1],errors='ignore')
                    msg_metadata[key] = hval
            except Exception as err:
                print("Could not decode headers, ignoring..: %s" % err)
        if not msg_metadata.get('message-id'):
            msg_metadata['message-id'] = mid
        mdate = None
        try:
            mdate = email.utils.parsedate_tz(msg_metadata.get('date'))
        except:
            pass
        if not mdate and msg_metadata.get('archived-at'):
            mdate = email.utils.parsedate_tz(msg_metadata.get('archived-at'))
        elif not mdate:
            print("Date seems totally wrong, setting to _now_ instead.")
            mdate = time.gmtime()
        mdatestring = time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(email.utils.mktime_tz(mdate)))
        body = self.msgbody(msg)
        try:
            if 'content-type' in msg_metadata and msg_metadata['content-type'].find("flowed") != -1:
                body = convertToWrapped(body, character_set="utf-8")
            if isinstance(body, str):
                body = body.encode('utf-8')
        except Exception as err:
            try:
                body = body.decode(chardet.detect(body)['encoding'])
            except Exception as err:
                try:
                    body = body.decode('latin-1')
                except:
                    try:
                        if isinstance(body, str):
                            body = body.encode('utf-8')
                    except:
                        body = None
        if body:
            attachments, contents = self.msgfiles(msg)
            private = False
            if hasattr(mlist, 'archive_public') and mlist.archive_public == True:
                private = False
            elif hasattr(mlist, 'archive_public') and mlist.archive_public == False:
                private = True
            elif hasattr(mlist, 'archive_policy') and mlist.archive_policy is not ArchivePolicy.public:
                private = True
            pmid = mid
            try:
                mid = "%s@%s@%s" % (hashlib.sha224(body if type(body) is bytes else body.encode('ascii', errors='ignore')).hexdigest(), email.utils.mktime_tz(mdate), lid)
            except Exception as err:
                if logger:
                    logger.warn("Could not generate MID: %s" % err)
                mid = pmid
            irt = ""
            if 'in-reply-to' in msg_metadata:
                try:
                    try:
                        irt = "\n".join(msg_metadata['in-reply-to'])
                    except:
                        irt = msg_metadata.get('in-reply-to').__str__()
                except:
                    irt = ""
            ojson = {
                'from_raw': msg_metadata['from'],
                'from': msg_metadata['from'],
                'to': msg_metadata['to'],
                'subject': msg_metadata['subject'],
                'message-id': msg_metadata['message-id'],
                'mid': mid,
                'cc': msg_metadata.get('cc'),
                'epoch': email.utils.mktime_tz(mdate),
                'list': lid,
                'list_raw': lid,
                'date': mdatestring,
                'private': private,
                'references': msg_metadata['references'],
                'in-reply-to': irt,
                'body': body.decode('utf-8', errors='replace') if type(body) is bytes else body,
                'attachments': attachments
            }
            
            if contents:
                for key in contents:
                    self.es.index(
                        index=self.dbname,
                        doc_type="attachment",
                        id=key,
                        body = {
                            'source': contents[key]
                        }
                    )
        
            self.es.index(
                index=self.dbname,
                doc_type="mbox",
                id=mid,
                body = ojson
            )
            
            self.es.index(
                index=self.dbname,
                doc_type="mbox_source",
                id=mid,
                body = {
                    "message-id": msg_metadata['message-id'],
                    "source": msg.as_string()
                }
            )
            
            # If MailMan and list info is present, save/update it in ES:
            if hasattr(mlist, 'description') and hasattr(mlist, 'list_name') and mlist.description and mlist.list_name:
                self.es.index(
                    index=self.dbname,
                    doc_type="mailinglists",
                    id=lid,
                    body = {
                        'list': lid,
                        'name': mlist.list_name,
                        'description': mlist.description,
                        'private': private
                    }
                )
            
            if logger:
                logger.info("Pony Mail archived message %s successfully" % mid)
            oldrefs = []
            
            # Is this a direct reply to a pony mail email?
            if irt != "":
                dm = re.search(r"pony-([a-f0-9]+)-([a-f0-9]+)@", irt)
                if dm:
                    cid = dm.group(1)
                    mid = dm.group(2)
                    if self.es.exists(index = self.dbname, doc_type = 'account', id = cid):
                        doc = self.es.get(index = self.dbname, doc_type = 'account', id = cid)
                        if doc:
                            oldrefs.append(cid)
                            self.es.index(
                                index=indexname,
                                doc_type="notifications",
                                body = {
                                    'type': 'direct',
                                    'recipient': cid,
                                    'list': lid,
                                    'private': private,
                                    'date': msg_metadata['date'],
                                    'from': msg_metadata['from'],
                                    'to': msg_metadata['to'],
                                    'subject': msg_metadata['subject'],
                                    'message-id': msg_metadata['message-id'],
                                    'in-reply-to': irt,
                                    'epoch': email.utils.mktime_tz(mdate),
                                    'mid': mid,
                                    'seen': 0
                                }
                            )
                            if logger:
                                logger.info("Notification sent to %s for %s" % (cid, mid))

            # Are there indirect replies to pony emails?
            if msg_metadata.get('references'):
                for im in re.finditer(r"pony-([a-f0-9]+)-([a-f0-9]+)@", msg_metadata.get('references')):
                    cid = im.group(1)
                    mid = im.group(2)
                    if self.es.exists(index = self.dbname, doc_type = 'account', id = cid):
                        doc = self.es.get(index = self.dbname, doc_type = 'account', id = cid)
                        
                        # does the user want to be notified of indirect replies?
                        if doc and 'preferences' in doc['_source'] and doc['_source']['preferences'].get('notifications') == 'indirect' and not cid in oldrefs:
                            oldrefs.append(cid)
                            self.es.index(
                                index=self.dbname,
                                doc_type="notifications",
                                body = {
                                    'type': 'indirect',
                                    'recipient': cid,
                                    'list': lid,
                                    'private': private,
                                    'date': msg_metadata['date'],
                                    'from': msg_metadata['from'],
                                    'to': msg_metadata['to'],
                                    'subject': msg_metadata['subject'],
                                    'message-id': msg_metadata['message-id'],
                                    'in-reply-to': mirt,
                                    'epoch': email.utils.mktime_tz(mdate),
                                    'mid': mid,
                                    'seen': 0
                                }
                            )
                            if logger:
                                logger.info("Notification sent to %s for %s" % (cid, mid))
        return lid
            
    def list_url(self, mlist):
        """ Gots
            to
            be
            here
        """
        return None

    def permalink(self, mlist, msg):
        """ Gots
            to
            be
            here
        """
        return None
    
if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Command line options.')
    parser.add_argument('--altheader', dest='altheader', type=str, nargs=1,
                       help='Alternate header for list ID')
    parser.add_argument('--allowfrom', dest='allowfrom', type=str, nargs=1,
                       help='(optional) source IP (mail server) to allow posts from, ignore if no match')
    parser.add_argument('--ignore', dest='ignorefrom', type=str, nargs=1,
                       help='Sender/list to ignore input from (owner etc)')
    parser.add_argument('--private', dest='private', action='store_true', 
                       help='This is a private archive')
    parser.add_argument('--makedate', dest='makedate', action='store_true', 
                       help='Use the archive timestamp as the email date instead of the Date header')
    parser.add_argument('--quiet', dest='quiet', action='store_true', 
                       help='Do not exit -1 if the email could not be parsed')
    args = parser.parse_args()
    
    foo = Archiver()
    try:
        msg = email.message_from_file(sys.stdin)
        # We're reading from STDIN, so let's fake an MM3 call
        ispublic = True
        ignorefrom = None
        allowfrom = None
        if args.altheader:
            altheader = args.altheader[0]
            if altheader in msg:
                msg.add_header('list-id', msg.get(altheader))
        elif 'altheader' in sys.argv:
            altheader = sys.argv[len(sys.argv)-1]
            if altheader in msg:
                msg.add_header('list-id', msg.get(altheader))
                
        #Ignore based on --ignore flag?
        if args.ignorefrom:
            ignorefrom = args.ignorefrom[0]
            if fnmatch.fnmatch(msg.get("from"), ignorefrom) or (msg.get("list-id") and fnmatch.fnmatch(msg.get("list-id"), ignorefrom)):
                print("Ignoring message as instructed by --ignore flag")
                sys.exit(0)
        
        # Check CIDR if need be
        if args.allowfrom:
            from netaddr import IPNetwork, IPAddress
            c = IPNetwork(args.allowfrom[0])
            good = False
            for line in msg.get_all('received') or []:
                m = re.search(r"from .+\[(.+)\]", line)
                if m:
                    try:
                        ip = IPAddress(m.group(1))
                        if ip in c:
                            good = True
                            msg.add_header("ip-whitelisted", "yes")
                            break
                    except:
                        pass
            if not good:
                print("No whitelisted IP found in message, aborting")
                sys.exit(-1)
        # Replace date header with $now?
        if args.makedate:
            msg.replace_header('date', email.utils.formatdate())
            
        if args.private:
            ispublic = False
        if 'list-id' in msg:
            if not msg.get('archived-at'):
                msg.add_header('archived-at', email.utils.formatdate())
            msg_metadata = namedtuple('importmsg', ['list_id', 'archive_public'])(list_id = msg.get('list-id'), archive_public=ispublic)
            
            try:
                lid = foo.archive_message(msg_metadata, msg)
                print("%s: Done archiving to %s!" % (email.utils.formatdate(), lid))
            except Exception as err:
                print("Archiving failed!: %s" % err)
                raise Exception("Archiving to ES failed")
        else:
            print("Nothing to import (no list-id found!)")
    except Exception as err:
        if args.quiet:
            print("Could not parse email, but exiting quietly as --quiet is on: %s" % err)
        else:
            print("Could not parse email: %s" % err)
            sys.exit(-1)