#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
 #the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
This is a Kibble scanner plugin for Apache Pony Mail sources.
"""
import requests
import json
import time
import re
import hashlib
import sys
import datetime
import plugins.utils.jsonapi

title = "Scanner plugin for Apache Pony Mail"
version = "0.1.0"

def accepts(source):
    """ Test if source matches a Pony Mail archive """
    # If the source equals the plugin name, assume a yes
    if source['type'] == 'ponymail':
        return True
    
    # If it's of type 'mail', check the URL
    if source['type'] == 'mail':
        if re.match(r"(https?://.+)/list\.html\?(.+)@(.+)", source['sourceURL']):
            return True
    
    # Default to not recognizing the source
    return False

def countSubs(struct, kids = 0):
    """ Counts replies in a thread """
    if 'children' in struct and len(struct['children']) > 0:
        for child in struct['children']:
            kids += 1
            kids += countSubs(child)
    return kids

def repliedTo(emails, struct):
    myList = {}
    for eml in struct:
        myID = eml['tid']
        if 'children' in eml:
            for child in eml['children']:
                myList[child['tid']] = myID
                if len(child['children']) > 0:
                    cList = repliedTo(emails, child['children'])
                    myList.update(cList)
    return myList

def getSender(email):
    sender = email['from']
    name = sender
    m = re.match(r"(.+)\s*<(.+)>", email['from'], flags=re.UNICODE)
    if m:
        name = m.group(1).replace('"', "").strip()
        sender = m.group(2)
    return sender

def scan(KibbleBit, source):
    # Validate URL first
    url = re.match(r"(https?://.+)/list\.html\?(.+)@(.+)", source['sourceURL'])
    if not url:
        KibbleBit.pprint("Malformed or invalid Pony Mail URL passed to scanner: %s" % source['sourceURL'])
        source['steps']['mail'] = {
            'time': time.time(),
            'status': 'Could not parse Pony Mail URL!',
            'running': False,
            'good': False
        }
        KibbleBit.updateSource(source)
        return
    
    # Pony Mail requires a UI cookie in order to work. Maked sure we have one!
    cookie = None
    if 'creds' in source and source['creds']:
        cookie = source['creds'].get('cookie', None)
    if not cookie:
        KibbleBit.pprint("Pony Mail instance at %s requires an authorized cookie, none found! Bailing." % source['sourceURL'])
        source['steps']['mail'] = {
            'time': time.time(),
            'status': 'No authorized cookie found in source object.',
            'running': False,
            'good': False
        }
        KibbleBit.updateSource(source)
        return
    
    # Notify scanner and DB that this is valid and we've begun parsing
    KibbleBit.pprint("%s is a valid Pony Mail address, parsing" % source['sourceURL'])
    source['steps']['mail'] = {
        'time': time.time(),
        'status': 'Downloading Pony Mail statistics',
        'running': True,
        'good': True
    }
    KibbleBit.updateSource(source)
    
    
    # Get base URL, list and domain to parse
    u = url.group(1)
    l = url.group(2)
    d = url.group(3)
    
    # Get this month
    dt = time.gmtime(time.time())
    firstYear = 1970
    year = dt[0]
    month = dt[1]
    if month <= 0:
        month += 12
        year -= 1
    months = 0
    
    # Hash for keeping records of who we know
    knowns = {}
    
    # While we have older archives, continue to parse
    while firstYear <= year:
        statsurl = "%s/api/stats.lua?list=%s&domain=%s&d=%s" % (u, l, d, "%04u-%02u" % (year, month))
        dhash = hashlib.sha224((("%s %s") % (source['organisation'], statsurl)).encode('ascii', errors='replace')).hexdigest()
        found = False        
        if KibbleBit.exists('mailstats', dhash):
            found = True
        if months <= 1 or not found: # Always parse this month's stats :)
            months += 1
            KibbleBit.pprint("Parsing %04u-%02u" % (year, month))
            KibbleBit.pprint(statsurl)
            pd = datetime.date(year, month, 1).timetuple()
            try:
                js = plugins.utils.jsonapi.get(statsurl, cookie = cookie)
            except Exception as err:
                KibbleBit.pprint("Server error, skipping this month")
                month -= 1            
                if month <= 0:
                    month += 12
                    year -= 1
                continue
            if 'firstYear' in js:
                firstYear = js['firstYear']
                #print("First Year is %u" % firstYear)
            else:
                KibbleBit.pprint("JSON was missing fields, aborting!")
                break
            replyList = repliedTo(js['emails'], js['thread_struct'])
            topics = js['no_threads']
            posters = {}
            no_posters = 0
            emails = len(js['emails'])
            top10 = []
            for eml in js['thread_struct']:
                count = countSubs(eml, 0)
                subject = ""
                for reml in js['emails']:
                    if reml['id'] == eml['tid']:
                        subject = reml['subject']
                        break
                if len(subject) > 0 and count > 0:
                    subject = re.sub(r"^((re|fwd|aw|fw):\s*)+", "", subject, flags=re.IGNORECASE)
                    subject = re.sub(r"[\r\n\t]+", "", subject, count=20)
                    emlid = hashlib.sha1(subject.encode('ascii', errors='replace')).hexdigest()
                    top10.append([emlid, subject, count])
            i = 0
            for top in reversed(sorted(top10, key= lambda x: x[2])):
                i += 1
                if i > 10:
                    break
                KibbleBit.pprint("Found top 10: %s (%s emails)" % (top[1], top[2]))
                md = time.strftime("%Y/%m/%d %H:%M:%S", pd)
                mlhash = hashlib.sha224(( ("%s%s%s%s") % (top[0], source['sourceURL'], source['organisation'], md)).encode('ascii', errors='replace')).hexdigest() # one unique id per month per mail thread
                jst = {
                    'organisation': source['organisation'],
                    'sourceURL': source['sourceURL'],
                    'sourceID': source['sourceID'],
                    'date': md,
                    'emails': top[2],
                    'shash': top[0],
                    'subject': top[1],
                    'ts': time.mktime(pd),
                    'id': mlhash
                }
                KibbleBit.index('mailtop', mlhash, jst)
                
            for email in js['emails']:
                sender = email['from']
                name = sender
                m = re.match(r"(.+)\s*<(.+)>", email['from'], flags=re.UNICODE)
                if m:
                    name = m.group(1).replace('"', "").strip()
                    sender = m.group(2)
                if not sender in posters:
                    posters[sender] = {
                        'name': name,
                        'email': sender
                    }
                if not sender in knowns:
                    sid = hashlib.sha1( ("%s%s" % (source['organisation'], sender)).encode('ascii', errors='replace')).hexdigest()
                    if KibbleBit.exists('person',sid):
                        knowns[sender] = True
                if not sender in knowns or name != sender:
                    KibbleBit.append('person', 
                        {
                        'upsert': True,
                        'name': name,
                        'email': sender,
                        'organisation': source['organisation'],
                        'id' :hashlib.sha1( ("%s%s" % (source['organisation'], sender)).encode('ascii', errors='replace')).hexdigest()
                    })
                    knowns[sender] = True
                replyTo = None
                if email['id'] in replyList:
                    rt = replyList[email['id']]
                    for eml in js['emails']:
                        if eml['id'] == rt:
                            replyTo = getSender(eml)
                            print("Email was reply to %s" % sender)
                jse = {
                    'organisation': source['organisation'],
                    'sourceURL': source['sourceURL'],
                    'sourceID': source['sourceID'],
                    'date': time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(email['epoch'])),
                    'sender': sender,
                    'address': sender,
                    'subject': email['subject'],
                    'replyto': replyTo,
                    'ts': email['epoch'],
                    'id': email['id'],
                    'upsert': True
                }
                KibbleBit.append('email', jse)
            for sender in posters:
                no_posters += 1
            
            
            jso = {
                'organisation': source['organisation'],
                'sourceURL': source['sourceURL'],
                'sourceID': source['sourceID'],
                'date': time.strftime("%Y/%m/%d %H:%M:%S", pd),
                'authors': no_posters,
                'emails': emails,
                'topics': topics
            }
            #print("Indexing as %s" % dhash)
            KibbleBit.index('mailstats', dhash, jso)
        month -= 1            
        if month <= 0:
            month += 12
            year -= 1
        
    
    source['steps']['mail'] = {
        'time': time.time(),
        'status': 'Mail archives successfully scanned at ' + time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(time.time())),
        'running': False,
        'good': True
    }
    KibbleBit.updateSource(source)