| #!/usr/bin/env python3 |
| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| import argparse |
| import importlib.util |
| import logging |
| import os.path |
| import shutil |
| import sys |
| import yaml |
| |
| # Needs 3.4 or higher to work |
| if sys.version_info <= (3, 3): |
| print("This script requires Python 3.4 or higher in order to work!") |
| sys.exit(-1) |
| |
| # Backend needs 3.8 or higher, warn if not found. |
| if sys.version_info < (3, 7, 3): |
| print( |
| "Warning: Pony Mail Foal requires Python 3.7.3 or higher for backend operations." |
| ) |
| print( |
| "You will be able to run the setup using this version (%u.%u), but will need >=3.7.3" |
| % (sys.version_info.major, sys.version_info.minor) |
| ) |
| print("for operating the UI backend server.") |
| |
| DEFAULT_DB_URL = "http://localhost:9200/" |
| dburl = "" |
| dbname = "" |
| mlserver = "" |
| mldom = "" |
| wc = "" |
| genname = "" |
| wce = False |
| shards = 0 |
| replicas = -1 |
| nonce = None |
| supported_generators = ["dkim", "full"] |
| |
| |
| def create_indices(): |
| """Creates new indices for a fresh pony mail installation, it possible""" |
| # Check if index already exists |
| if es.indices.exists(dbname + "-mbox"): |
| if args.soe: |
| print( |
| "ElasticSearch indices with prefix '%s' already exists and SOE set, exiting quietly" |
| % dbname |
| ) |
| sys.exit(0) |
| else: |
| print( |
| "Error: Existing ElasticSearch indices with prefix '%s' already exist!" |
| % dbname |
| ) |
| sys.exit(-1) |
| |
| print(f"Creating indices {dbname}-*...") |
| |
| settings = {"number_of_shards": shards, "number_of_replicas": replicas} |
| mapping_file = yaml.safe_load(open("mappings.yaml", "r")) |
| for index, mappings in mapping_file.items(): |
| res = es.indices.create( |
| index=f"{dbname}-{index}", body={"mappings": mappings, "settings": settings} |
| ) |
| |
| print(f"Index {dbname}-{index} created! %s " % res) |
| |
| |
| # Check for all required Python packages |
| wanted_pkgs = [ |
| "elasticsearch", # used by setup.py, archiver.py and elastic.py |
| "formatflowed", # used by archiver.py |
| "netaddr", # used by archiver.py |
| "certifi", # used by archiver.py and elastic.py |
| ] |
| |
| missing_pkgs = list(wanted_pkgs) # copy to avoid corruption |
| for pkg in wanted_pkgs: |
| if importlib.util.find_spec(pkg): |
| missing_pkgs.remove(pkg) |
| |
| if missing_pkgs: |
| print("It looks like you need to install some Python modules first") |
| print("The following packages are required: ") |
| for pkg in missing_pkgs: |
| print(" - %s" % pkg) |
| print("You may use your package manager, or run the following command:") |
| print("pip3 install %s" % " ".join(missing_pkgs)) |
| sys.exit(-1) |
| |
| |
| # at this point we can assume elasticsearch is present |
| from elasticsearch import VERSION as ES_VERSION |
| from elasticsearch import ConnectionError as ES_ConnectionError |
| from elasticsearch import Elasticsearch, ElasticsearchException |
| |
| ES_MAJOR = ES_VERSION[0] |
| |
| # CLI arg parsing |
| parser = argparse.ArgumentParser(description="Command line options.") |
| |
| parser.add_argument( |
| "--defaults", dest="defaults", action="store_true", help="Use default settings" |
| ) |
| parser.add_argument( |
| "--devel", dest="devel", action="store_true", help="Use developer settings (shards=1, replicas=0)" |
| ) |
| parser.add_argument( |
| "--clobber", |
| dest="clobber", |
| action="store_true", |
| help="Allow overwrite of ponymail.yaml & ../site/api/lib/config.lua (default: create *.tmp if either exists)", |
| ) |
| parser.add_argument("--dburl", dest="dburl", type=str, help="ES backend URL") |
| parser.add_argument("--dbname", dest="dbname", type=str, help="ES DB prefix") |
| parser.add_argument("--dbshards", dest="dbshards", type=int, help="DB Shard Count") |
| parser.add_argument( |
| "--dbreplicas", dest="dbreplicas", type=int, help="DB Replica Count" |
| ) |
| parser.add_argument( |
| "--mailserver", |
| dest="mailserver", |
| type=str, |
| help="Host name of outgoing mail server", |
| ) |
| parser.add_argument( |
| "--mldom", dest="mldom", type=str, help="Domains to accept mail for via UI" |
| ) |
| parser.add_argument( |
| "--wordcloud", dest="wc", action="store_true", help="Enable word cloud" |
| ) |
| parser.add_argument( |
| "--skiponexist", |
| dest="soe", |
| action="store_true", |
| help="Skip setup if ES index exists", |
| ) |
| parser.add_argument( |
| "--noindex", |
| dest="noi", |
| action="store_true", |
| help="Don't create ElasticSearch indices, assume they exist", |
| ) |
| parser.add_argument( |
| "--nocloud", dest="nwc", action="store_true", help="Do not enable word cloud" |
| ) |
| parser.add_argument( |
| "--generator", |
| dest="generator", |
| type=str, |
| help="Document ID Generator to use (dkim, full)", |
| ) |
| parser.add_argument( |
| "--nonce", |
| dest="nonce", |
| type=str, |
| help="Cryptographic nonce to use if generator is DKIM/RFC-6376 (--generator dkim)", |
| ) |
| args = parser.parse_args() |
| |
| print("") |
| print("Welcome to the Pony Mail setup script!") |
| print("Let's start by determining some settings...") |
| print("") |
| |
| |
| # If called with --defaults (like from Docker), use default values |
| if args.defaults: |
| dburl = DEFAULT_DB_URL |
| dbname = "ponymail" |
| mlserver = "localhost" |
| mldom = "example.org" |
| wc = "Y" |
| wce = True |
| shards = 3 |
| replicas = 1 |
| genname = "dkim" |
| urlPrefix = "" |
| nonce = None |
| |
| if args.devel: |
| dburl = DEFAULT_DB_URL |
| dbname = "ponymail" |
| mlserver = "localhost" |
| mldom = "example.org" |
| wc = "Y" |
| wce = True |
| shards = 1 |
| replicas = 0 |
| genname = "dkim" |
| urlPrefix = "" |
| nonce = None |
| |
| # Accept CLI args, copy them |
| if args.dburl: |
| dburl = args.dburl |
| if args.dbname: |
| dbname = args.dbname |
| if args.mailserver: |
| mlserver = args.mailserver |
| if args.mldom: |
| mldom = args.mldom |
| if args.wc: |
| wc = args.wc |
| if args.nwc: |
| wc = "n" |
| wce = False |
| if args.dbshards: |
| shards = args.dbshards |
| if args.dbreplicas is not None: # Allow for 0 value |
| replicas = args.dbreplicas |
| if args.generator: |
| if all(x in supported_generators for x in args.generator.split(' ')): |
| genname = args.generator |
| else: |
| sys.stderr.write( |
| "Invalid generator specified. Must be one of: " |
| + ", ".join(supported_generators) |
| + "\n" |
| ) |
| sys.exit(-1) |
| if args.generator and any(x == "dkim" for x in args.generator.split(' ')) and args.nonce is not None: |
| nonce = args.nonce |
| |
| if not dburl: |
| dburl = input("What is the URL of the ElasticSearch server? [%s]: " % DEFAULT_DB_URL) |
| if not dburl: |
| dburl = DEFAULT_DB_URL |
| |
| if not dbname: |
| dbname = input("What would you like to call the mail index [ponymail]: ") |
| if not dbname: |
| dbname = "ponymail" |
| |
| if not mlserver: |
| mlserver = input( |
| "What is the hostname of the outgoing mailserver hostname? [localhost]: " |
| ) |
| if not mlserver: |
| mlserver = "localhost" |
| |
| if not mldom: |
| mldom = input("Which domains would you accept mail to from web-replies? [*]: ") |
| if not mldom: |
| mldom = "*" |
| |
| while wc.lower() not in ["y", "n"]: |
| wc = input("Would you like to enable the word cloud feature? (Y/N) [Y]: ").lower() |
| if not wc: |
| wc = "y" |
| if wc.lower() == "y": |
| wce = True |
| |
| while genname == "": |
| print("Please select a document ID generator:") |
| print( |
| "1 [RECOMMENDED] DKIM/RFC-6376: Short SHA3 hash useful for cluster setups with permalink usage" |
| ) |
| print( |
| "2 FULL: Full message digest with MTA trail. Not recommended for clustered setups." |
| ) |
| try: |
| ans = input("Please select a generator (1 or 2) [1]: ") |
| if ans: |
| gno = int(ans) |
| else: |
| gno = 1 |
| if gno <= len(supported_generators) and supported_generators[gno - 1]: |
| genname = supported_generators[gno - 1] |
| except ValueError: |
| pass |
| |
| if genname == "dkim" and (nonce is None and not args.defaults and not args.devel): |
| print( |
| "DKIM hasher chosen. It is recommended you set a cryptographic nonce for this generator, though not required." |
| ) |
| print( |
| "If you set a nonce, you will need this same nonce for future installations if you intend to preserve " |
| ) |
| print("permalinks from imported messages.") |
| nonce = ( |
| input("Enter your nonce or hit [enter] to continue without a nonce: ") or None |
| ) |
| |
| while shards < 1: |
| try: |
| ans = input("How many shards for the ElasticSearch index? [3]: ") |
| if ans: |
| shards = int(ans) |
| else: |
| shards = 3 |
| except ValueError: |
| pass |
| |
| while replicas < 0: |
| try: |
| ans = input("How many replicas for each shard? [1]: ") |
| if ans: |
| replicas = int(ans) |
| else: |
| replicas = 1 |
| except ValueError: |
| pass |
| |
| print("Okay, I got all I need, setting up Pony Mail...") |
| |
| # we need to connect to database to determine the engine version |
| es = Elasticsearch( |
| [dburl], |
| max_retries=5, |
| retry_on_timeout=True, |
| ) |
| |
| # elasticsearch logs lots of warnings on retries/connection failure |
| logging.getLogger("elasticsearch").setLevel(logging.ERROR) |
| |
| try: |
| DB_VERSION = es.info()["version"]["number"] |
| except ES_ConnectionError: |
| print("WARNING: Connection error: could not determine the engine version.") |
| DB_VERSION = "0.0.0" |
| |
| DB_MAJOR = int(DB_VERSION.split(".")[0]) |
| print( |
| "Versions: library %d (%s), engine %d (%s)" |
| % (ES_MAJOR, ".".join(map(str, ES_VERSION)), DB_MAJOR, DB_VERSION) |
| ) |
| if DB_MAJOR < 7: |
| print("This version of Pony Mail requires ElasticSearch 7.x or higher") |
| |
| if not DB_MAJOR == ES_MAJOR: |
| print("WARNING: library version does not agree with engine version!") |
| |
| if DB_MAJOR == 0: # not known |
| if args.noi: |
| # allow setup to be used without engine running |
| print( |
| "Could not determine the engine version. Assume it is the same as the library version." |
| ) |
| DB_MAJOR = ES_MAJOR |
| else: |
| # if we cannot connect to get the version, we cannot create the index later |
| print("Could not connect to the engine. Fatal.") |
| sys.exit(1) |
| |
| if not args.noi: |
| try: |
| create_indices() |
| except ElasticsearchException as e: |
| print("Index creation failed: %s" % e) |
| sys.exit(1) |
| |
| ponymail_cfg = "archiver.yaml" |
| if not args.clobber and os.path.exists(ponymail_cfg): |
| print("%s exists and clobber is not set" % ponymail_cfg) |
| ponymail_cfg = "archiver.yaml.tmp" |
| |
| print("Writing importer config (%s)" % ponymail_cfg) |
| |
| with open(ponymail_cfg, "w") as f: |
| f.write( |
| """ |
| --- |
| ############################################################### |
| # An archiver.yaml is needed to run this project. This sample config file was |
| # originally generated by tools/setup.py. |
| # |
| # Run the tools/setup.py script and an archiver.yaml which looks a lot like this |
| # one will be generated. If, for whatever reason, that script is not working |
| # for you, you may use this archiver.yaml as a starting point. |
| # |
| # Contributors should strive to keep this sample updated. One way to do this |
| # would be to run the tools/setup.py, rename the generated config to |
| # archiver.yaml.sample, and then pasting this message or a modified form of |
| # this message at the top. |
| ############################################################### |
| |
| ############################################################### |
| # Pony Mail Archiver Configuration file |
| |
| |
| # Main ES configuration |
| elasticsearch: |
| dburl: %s |
| dbname: %s |
| #wait: active shard count |
| #backup: database name |
| |
| archiver: |
| #generator: dkim|full (dkim recommended) |
| generator: %s |
| nonce: %s |
| policy: default # message parsing policy: default, compat32, smtputf8 |
| |
| debug: |
| #cropout: string to crop from list-id |
| |
| """ |
| % (dburl, dbname, genname, nonce or "~") |
| ) |
| |
| print("Copying sample JS config to config.js (if needed)...") |
| if not os.path.exists("../site/js/config.js") and os.path.exists( |
| "../site/js/config.js.sample" |
| ): |
| shutil.copy("../site/js/config.js.sample", "../site/js/config.js") |
| |
| server_cfg = "../server/ponymail.yaml" |
| if not args.clobber and os.path.exists(server_cfg): |
| print("%s exists and clobber is not set" % server_cfg) |
| server_cfg = "../server/ponymail.yaml.tmp" |
| |
| print("Writing UI backend configuration file %s" % server_cfg) |
| with open(server_cfg, "w") as f: |
| f.write(""" |
| server: |
| port: 8080 # Port to bind to |
| bind: 127.0.0.1 # IP to bind to - typically 127.0.0.1 for localhost or 0.0.0.0 for all IPs |
| |
| |
| database: |
| dburl: %s # The URL of the ElasticSearch database |
| db_prefix: %s # DB prefix, usually 'ponymail' |
| max_hits: 15000 # Maximum number of emails to process in a search |
| pool_size: 15 # number of connections for async queries |
| max_lists: 8192 # max number of lists to allow for |
| |
| ui: |
| wordcloud: %s |
| mailhost: %s |
| sender_domains: "%s" |
| traceback: true |
| mgmtconsole: true # enable email admin |
| true_gdpr: true # fully delete emails instead of marking them deleted |
| |
| tasks: |
| refresh_rate: 150 # Background indexer run interval, in seconds |
| |
| # Fill in OAuth data as needed |
| oauth: |
| # If using OAuth, set the authoritative domains here. These are the OAuth domains that |
| # will provide access to private emails. |
| # authoritative_domains: |
| # - googleapis.com # OAuth via google is authoritative |
| # - github.com # GitHub OAuth is authoritative |
| # admins: |
| # - foo@example.org |
| google_client_id: ~ |
| github_client_id: ~ |
| github_client_secret: ~ |
| |
| """ % (dburl, dbname, "true" if wce else "false", mlserver, mldom)) |
| |
| |
| print("All done, Pony Mail should...work now :)") |
| print( |
| "If you are using an external mail inbound server, \nmake sure to copy the contents of this tools directory to it" |
| ) |