| #!/usr/bin/env python3 |
| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| import argparse |
| import importlib.util |
| import logging |
| import os |
| import os.path |
| import shutil |
| import sys |
| |
| if sys.version_info <= (3, 3): |
| print("This script requires Python 3.4 or higher") |
| sys.exit(-1) |
| |
| # Check for all required python packages |
| wanted_pkgs = [ |
| 'elasticsearch',# used by setup.py, archiver.py and elastic.py |
| 'formatflowed', # used by archiver.py |
| 'netaddr', # used by archiver.py |
| 'certifi', # used by archiver.py and elastic.py |
| ] |
| |
| missing_pkgs = list(wanted_pkgs) # copy to avoid corruption |
| for pkg in wanted_pkgs: |
| if importlib.util.find_spec(pkg): |
| missing_pkgs.remove(pkg) |
| |
| if missing_pkgs: |
| print("It looks like you need to install some python modules first") |
| print("The following packages are required: ") |
| for pkg in missing_pkgs: |
| print(" - %s" % pkg) |
| print("You may use your package manager, or run the following command:") |
| print("pip3 install %s" % " ".join(missing_pkgs)) |
| sys.exit(-1) |
| |
| |
| # at this point we can assume elasticsearch is present |
| from elasticsearch import Elasticsearch |
| from elasticsearch import ElasticsearchException |
| from elasticsearch import ConnectionError as ES_ConnectionError |
| from elasticsearch import VERSION as ES_VERSION |
| ES_MAJOR = ES_VERSION[0] |
| |
| # CLI arg parsing |
| parser = argparse.ArgumentParser(description='Command line options.') |
| |
| parser.add_argument('--defaults', dest='defaults', action='store_true', |
| help='Use default settings') |
| parser.add_argument('--dbprefix', dest='dbprefix') |
| parser.add_argument('--clobber', dest='clobber', action='store_true', |
| help='Allow overwrite of ponymail.cfg & ../site/api/lib/config.lua (default: create *.tmp if either exists)') |
| parser.add_argument('--dbhost', dest='dbhost', type=str, |
| help='ES backend hostname') |
| parser.add_argument('--dbport', dest='dbport', type=str, |
| help='DB port') |
| parser.add_argument('--dbname', dest='dbname', type=str, |
| help='ES DB name') |
| parser.add_argument('--dbshards', dest='dbshards', type=int, |
| help='DB Shard Count') |
| parser.add_argument('--dbreplicas', dest='dbreplicas', type=int, |
| help='DB Replica Count') |
| parser.add_argument('--mailserver', dest='mailserver', type=str, |
| help='Host name of outgoing mail server') |
| parser.add_argument('--mldom', dest='mldom', type=str, |
| help='Domains to accept mail for via UI') |
| parser.add_argument('--wordcloud', dest='wc', action='store_true', |
| help='Enable word cloud') |
| parser.add_argument('--skiponexist', dest='soe', action='store_true', |
| help='Skip setup if ES index exists') |
| parser.add_argument('--noindex', dest='noi', action='store_true', |
| help="Don't make an ES index, assume it exists") |
| parser.add_argument('--nocloud', dest='nwc', action='store_true', |
| help='Do not enable word cloud') |
| parser.add_argument('--generator', dest='generator', type=str, |
| help='Document ID Generator to use (legacy, medium, cluster, full)') |
| args = parser.parse_args() |
| |
| print("Welcome to the Pony Mail setup script!") |
| print("Let's start by determining some settings...") |
| print("") |
| |
| |
| hostname = "" |
| port = 0 |
| dbname = "" |
| mlserver = "" |
| mldom = "" |
| wc = "" |
| genname = "" |
| wce = False |
| shards = 0 |
| replicas = -1 |
| urlPrefix = None |
| |
| # If called with --defaults (like from Docker), use default values |
| if args.defaults: |
| hostname = "localhost" |
| port = 9200 |
| dbname = "ponymail" |
| mlserver = "localhost" |
| mldom = "example.org" |
| wc = "Y" |
| wce = True |
| shards = 1 |
| replicas = 0 |
| genname = "cluster" |
| urlPrefix = '' |
| |
| # Accept CLI args, copy them |
| if args.dbprefix: |
| urlPrefix = args.dbprefix |
| if args.dbhost: |
| hostname = args.dbhost |
| if args.dbport: |
| port = int(args.dbport) |
| if args.dbname: |
| dbname = args.dbname |
| if args.mailserver: |
| mlserver = args.mailserver |
| if args.mldom: |
| mldom = args.mldom |
| if args.wc: |
| wc = args.wc |
| if args.nwc: |
| wc = False |
| if args.dbshards: |
| shards = args.dbshards |
| if args.dbreplicas: |
| replicas = args.dbreplicas |
| if args.generator: |
| genname = args.generator |
| |
| while hostname == "": |
| hostname = input("What is the hostname of the ElasticSearch server? (e.g. localhost): ") |
| |
| while urlPrefix == None: |
| urlPrefix = input("Database URL prefix if any (hit enter if none): ") |
| |
| while port < 1: |
| try: |
| port = int(input("What port is ElasticSearch listening on? (normally 9200): ")) |
| except ValueError: |
| pass |
| |
| while dbname == "": |
| dbname = input("What would you like to call the mail index (e.g. ponymail): ") |
| |
| while mlserver == "": |
| mlserver = input("What is the hostname of the outgoing mailserver? (e.g. mail.foo.org): ") |
| |
| while mldom == "": |
| mldom = input("Which domains would you accept mail to from web-replies? (e.g. foo.org or *): ") |
| |
| while wc == "": |
| wc = input("Would you like to enable the word cloud feature? (Y/N): ") |
| if wc.lower() == "y": |
| wce = True |
| |
| while genname == "": |
| gens = ['legacy', 'medium', 'cluster', 'full'] |
| print ("Please select a document ID generator:") |
| print("1 LEGACY: The original document generator for v/0.1-0.8 (no longer recommended)") |
| print("2 MEDIUM: The medium comprehensive generator for v/0.9 (no longer recommended)") |
| print("3 REDUNDANT: Near-full message digest, discard MTA trail (recommended for clustered setups)") |
| print("4 FULL: Full message digest with MTA trail (recommended for single-node setups).") |
| try: |
| gno = int(input("Please select a generator [1-4]: ")) |
| if gno <= len(gens) and gens[gno-1]: |
| genname = gens[gno-1] |
| except ValueError: |
| pass |
| |
| while shards < 1: |
| try: |
| shards = int(input("How many shards for the ElasticSearch index? ")) |
| except ValueError: |
| pass |
| |
| while replicas < 0: |
| try: |
| replicas = int(input("How many replicas for each shard? ")) |
| except ValueError: |
| pass |
| |
| print("Okay, I got all I need, setting up Pony Mail...") |
| |
| def createIndex(): |
| # Check if index already exists |
| if es.indices.exists(dbname): |
| if args.soe: |
| print("ElasticSearch index '%s' already exists and SOE set, exiting quietly" % dbname) |
| sys.exit(0) |
| else: |
| print("Error: ElasticSearch index '%s' already exists!" % dbname) |
| sys.exit(-1) |
| |
| print("Creating index " + dbname) |
| |
| settings = { |
| "number_of_shards" : shards, |
| "number_of_replicas" : replicas |
| } |
| |
| mappings = { |
| "mbox" : { |
| "properties" : { |
| "@import_timestamp" : { |
| "type" : "date", |
| "format" : "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd" |
| }, |
| "attachments" : { |
| "properties" : { |
| "content_type" : { |
| "type" : "string", |
| "index" : "not_analyzed" |
| }, |
| "filename" : { |
| "type" : "string", |
| "index" : "not_analyzed" |
| }, |
| "hash" : { |
| "type" : "string", |
| "index" : "not_analyzed" |
| }, |
| "size" : { |
| "type" : "long" |
| } |
| } |
| }, |
| "body" : { |
| "type" : "string" |
| }, |
| "cc": { |
| "type": "string" |
| }, |
| "date" : { |
| "type" : "date", |
| "store" : True, |
| "format" : "yyyy/MM/dd HH:mm:ss", |
| "index" : "not_analyzed" |
| }, |
| "epoch" : { # number of seconds since the epoch |
| "type" : "long", |
| "index" : "not_analyzed" |
| }, |
| "from" : { |
| "type" : "string" |
| }, |
| "from_raw" : { |
| "type" : "string", |
| "index" : "not_analyzed" |
| }, |
| "in-reply-to" : { |
| "type" : "string", |
| "index" : "not_analyzed" |
| }, |
| "list" : { |
| "type" : "string" |
| }, |
| "list_raw" : { |
| "type" : "string", |
| "index" : "not_analyzed" |
| }, |
| "message-id" : { |
| "type" : "string", |
| "index" : "not_analyzed" |
| }, |
| "mid" : { |
| "type" : "string" |
| }, |
| "private" : { |
| "type" : "boolean" |
| }, |
| "references" : { |
| "type" : "string" |
| }, |
| "subject" : { |
| "type" : "string", |
| "fielddata": True # dropped later if DB_MAJOR==2 |
| }, |
| "to" : { |
| "type" : "string" |
| } |
| } |
| }, |
| "attachment" : { |
| "properties" : { |
| "source" : { |
| "type" : "binary" |
| } |
| } |
| }, |
| "mbox_source" : { |
| "_all": { |
| "enabled": False # this doc type is not searchable |
| }, |
| "properties" : { |
| "source" : { |
| "type" : "binary" |
| }, |
| "message-id" : { |
| "type" : "string", |
| "index" : "not_analyzed" |
| }, |
| "mid" : { |
| "type" : "string" |
| } |
| } |
| }, |
| "mailinglists" : { |
| "_all": { |
| "enabled": False # this doc type is not searchable |
| }, |
| "properties" : { |
| "description" : { |
| "type" : "string", |
| "index" : "not_analyzed" |
| }, |
| "list" : { |
| "type" : "string", |
| # "index" : "not_analyzed" |
| }, |
| "name" : { |
| "type" : "string", |
| "index" : "not_analyzed" |
| } |
| } |
| }, |
| "account" : { |
| "_all": { |
| "enabled": False # this doc type is not searchable |
| }, |
| "properties" : { |
| "cid" : { |
| "type" : "string", |
| "index" : "not_analyzed" |
| }, |
| "credentials" : { |
| "properties" : { |
| "altemail" : { |
| "type" : "object" |
| }, |
| "email" : { |
| "type" : "string", |
| "index" : "not_analyzed" |
| }, |
| "fullname" : { |
| "type" : "string", |
| "index" : "not_analyzed" |
| }, |
| "uid" : { |
| "type" : "string", |
| "index" : "not_analyzed" |
| } |
| } |
| }, |
| "internal" : { |
| "properties" : { |
| "cookie" : { |
| "type" : "string", |
| "index" : "not_analyzed" |
| }, |
| "ip" : { |
| "type" : "string", |
| "index" : "not_analyzed" |
| }, |
| "oauth_used" : { |
| "type" : "string", |
| "index" : "not_analyzed" |
| } |
| } |
| }, |
| "request_id" : { |
| "type" : "string", |
| "index" : "not_analyzed" |
| } |
| } |
| }, |
| "notifications" : { |
| "_all": { |
| "enabled": False # this doc type is not searchable |
| }, |
| "properties" : { |
| "date" : { |
| "type" : "date", |
| "store" : True, |
| "format" : "yyyy/MM/dd HH:mm:ss" |
| }, |
| "epoch" : { |
| "type" : "long" |
| }, |
| "from" : { |
| "type" : "string", |
| # "index" : "not_analyzed" |
| }, |
| "in-reply-to" : { |
| "type" : "string", |
| "index" : "not_analyzed" |
| }, |
| "list" : { |
| "type" : "string", |
| # "index" : "not_analyzed" |
| }, |
| "message-id" : { |
| "type" : "string", |
| "index" : "not_analyzed" |
| }, |
| "mid" : { |
| "type" : "string", |
| # "index" : "not_analyzed" |
| }, |
| "private" : { |
| "type" : "boolean" |
| }, |
| "recipient" : { |
| "type" : "string", |
| "index" : "not_analyzed" |
| }, |
| "seen" : { |
| "type" : "long" |
| }, |
| "subject" : { |
| "type" : "string", |
| "fielddata": True # dropped later if DB_MAJOR==2 |
| # "index" : "not_analyzed" |
| }, |
| "to" : { |
| "type" : "string", |
| # "index" : "not_analyzed" |
| }, |
| "type" : { |
| "type" : "string", |
| "index" : "not_analyzed" |
| } |
| } |
| } |
| } |
| |
| if DB_MAJOR == 2: # ES 2 handles fielddata differently |
| del mappings['mbox']['properties']['subject']['fielddata'] |
| del mappings['notifications']['properties']['subject']['fielddata'] |
| |
| res = es.indices.create(index = dbname, body = { |
| "mappings" : mappings, |
| "settings": settings |
| } |
| ) |
| |
| print("Index created! %s " % res) |
| |
| # we need to connect to database to determine the engine version |
| es = Elasticsearch([ |
| { |
| 'host': hostname, |
| 'port': port, |
| 'use_ssl': False, |
| 'url_prefix': urlPrefix |
| }], |
| max_retries=5, |
| retry_on_timeout=True |
| ) |
| |
| # elasticsearch logs lots of warnings on retries/connection failure |
| logging.getLogger("elasticsearch").setLevel(logging.ERROR) |
| |
| try: |
| DB_VERSION=es.info()['version']['number'] |
| except ES_ConnectionError: |
| print("WARNING: Connection error: could not determine the engine version.") |
| DB_VERSION='0.0.0' |
| |
| DB_MAJOR=int(DB_VERSION.split('.')[0]) |
| print("Versions: library %d (%s), engine %d (%s)" % (ES_MAJOR, '.'.join(map(str,ES_VERSION)) , DB_MAJOR, DB_VERSION)) |
| |
| if not DB_MAJOR == ES_MAJOR: |
| print("WARNING: library version does not agree with engine version!") |
| |
| if DB_MAJOR == 0: # not known |
| if args.noi: |
| # allow setup to be used without engine running |
| print("Could not determine the engine version. Assume it is the same as the library version.") |
| DB_MAJOR = ES_MAJOR |
| else: |
| # if we cannot connect to get the version, we cannot create the index later |
| print("Could not connect to the engine. Fatal.") |
| sys.exit(1) |
| |
| if not args.noi: |
| try: |
| createIndex() |
| except ElasticsearchException as e: |
| print("Index creation failed: %s" % e) |
| sys.exit(1) |
| |
| ponymail_cfg = 'ponymail.cfg' |
| if not args.clobber and os.path.exists(ponymail_cfg): |
| print("%s exists and clobber is not set" % ponymail_cfg) |
| ponymail_cfg = 'ponymail.cfg.tmp' |
| |
| print("Writing importer config (%s)" % ponymail_cfg) |
| |
| with open(ponymail_cfg, "w") as f: |
| f.write(""" |
| ############################################################### |
| # Pony Mail Configuration file |
| |
| # Main ES configuration |
| [elasticsearch] |
| hostname: %s |
| dbname: %s |
| port: %u |
| ssl: false |
| |
| #uri: url_prefix |
| |
| #user: username |
| #password: password |
| |
| #%s |
| |
| #backup: database name |
| |
| [archiver] |
| generator: %s |
| |
| [debug] |
| #cropout: string to crop from list-id |
| |
| ############################################################### |
| """ % (hostname, dbname, port, |
| 'wait: active shard count' if DB_MAJOR == 5 else 'write: consistency level (default quorum)', genname)) |
| |
| config_path = "../site/api/lib" |
| config_file = "config.lua" |
| if not args.clobber and os.path.exists(os.path.join(config_path,config_file)): |
| print("%s exists and clobber is not set" % config_file) |
| config_file = "config.lua.tmp" |
| print("mod_lua configuration (%s)" % config_file) |
| with open(os.path.join(config_path,config_file), "w") as f: |
| f.write(""" |
| local config = { |
| es_url = "http://%s:%u/%s/", |
| mailserver = "%s", |
| -- mailport = 1025, -- override the default port (25) |
| accepted_domains = "%s", |
| wordcloud = %s, |
| email_footer = nil, -- see the docs for how to set this up. |
| full_headers = false, |
| maxResults = 5000, -- max emails to return in one go. Might need to be bumped for large lists |
| -- stats_maxBody = 200, -- max size of body snippet returned by stats.lua |
| -- stats_wordExclude = ".|..|...", -- patterns to exclude from word cloud generated by stats.lua |
| admin_oauth = {}, -- list of domains that may do administrative oauth (private list access) |
| -- add 'www.googleapis.com' to the list for google oauth to decide, for instance. |
| oauth_fields = { -- used for specifying individual oauth handling parameters. |
| -- for example: |
| -- internal = { |
| -- email = 'CAS-EMAIL', |
| -- name = 'CAS-NAME', |
| -- uid = 'REMOTE-USER', |
| -- env = 'subprocess' -- use environment vars instead of request headers |
| -- } |
| }, |
| -- allow_insecure_cookie = true, -- override the default (false) - only use for test installations |
| -- no_association = {}, -- domains that are not allowed for email association |
| -- listsDisplay = 'regex', -- if defined, hide list names that don't match the regex |
| -- debug = false, -- whether to return debug information |
| antispam = true -- Whether or not to add anti-spam measures aimed at anonymous users. |
| } |
| return config |
| """ % (hostname, port, dbname, mlserver, mldom, "true" if wce else "false")) |
| |
| print("Copying sample JS config to config.js (if needed)...") |
| if not os.path.exists("../site/js/config.js") and os.path.exists("../site/js/config.js.sample"): |
| shutil.copy("../site/js/config.js.sample", "../site/js/config.js") |
| |
| |
| print("All done, Pony Mail should...work now :)") |
| print("If you are using an external mail inbound server, \nmake sure to copy archiver.py and ponymail.cfg to it") |
