diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml
index f8549ad..d78424e 100644
--- a/.github/workflows/pythonpackage.yml
+++ b/.github/workflows/pythonpackage.yml
@@ -14,18 +14,28 @@
 
     steps:
     - uses: actions/checkout@master
-    - uses: actions/checkout@master   
       with:
+        persist-credentials: false
+    - uses: actions/checkout@master
+      with:
+        persist-credentials: false
         repository: apache/incubator-ponymail-unit-tests
         path: pmtests
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v1
       with:
-        version: ${{ matrix.python-version }}
+        python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -r requirements.txt
+        pip install -r test/requirements.txt
+    - name: Set up LUA
+    # This is the commit for v8.0.0 (current at time of approval by INFRA)
+      uses: leafo/gh-actions-lua@ea0ae38722c0b45aa4e770f7c4a650c6b26800d0
+      with:
+        luaVersion: "5.2"
+    - name: Basic Test with LUA
+      run: lua -v
 #    - name: Test with pytest
 #      run: |
 #        pip install pytest
@@ -35,3 +45,7 @@
         sed -e 's/# cropout:/cropout:/' tools/ponymail.cfg.sample >tools/ponymail.cfg
         cd pmtests
         python runall.py --root ..
+    - name: Generator tests
+      run: |
+        cd test
+        python generatortest.py generatortest.yaml
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 218fe8e..b6cf115 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,12 @@
 ## Changes in 0.12:
+-  Bug: stats.lua dfrom and dto parameters are not documented in API (#544)
+- Bug: atom.lua is not documented (#538)
+- Bug: notifications.lua 'seen' option is not documented (#539)
+- Bug: &since option for stats.lua is not documented (#537)
+- Bug: PM setup does not support ES 6 - undocumented (#536)
+- Bug: buffering=1 not allowed with binary files (#534)
+- Bug: Invalid mailing list address supplied (#516)
+- Enh: stats.lua could return monthly stats to show in calendar (#532)
 - Bug: archiver.py msgbody() function issues (#244 and #463)
 - Bug: archiver.py: convertToWrapped expects bytes (#462)
 - Bug: convertToWrapped expects a bytestring (#518)
diff --git a/README.md b/README.md
index c5d4316..a82f0ff 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,5 @@
+_NB: Development focus has shifted towards our [Pony Mail codename Foal](https://github.com/apache/incubator-ponymail-foal) replacement suite.<br/>Suggestions for new features should be focused on that repository instead._
+
 # Apache Pony Mail (Incubating)
 
 **Development on this project has shifted to [incubator-ponymail-foal](https://github.com/apache/incubator-ponymail-foal)**. This repository is now deprecated.
@@ -15,7 +17,8 @@
 
 ![Trends](https://github.com/apache/incubator-ponymail/blob/master/site/images/demo_trends.png)
 
-See [https://lists.apache.org](https://lists.apache.org) for a demo.
+See [https://lists.apache.org](https://lists.apache.org) for a demo (this now runs the 
+[Foal](https://github.com/apache/incubator-ponymail-foal) suite).
 
 Pony Mail works in both public, private and mixed-mode, allowing you 
 to have one unified place for all your communication, both public and 
@@ -44,7 +47,7 @@
 ### Requirements: ###
 
 * Linux operating system (tested on Ubuntu, Debian, Fedora and CentOS - Windows or OS/X may work)
-* ElasticSearch backend (2.1 minimum)
+* ElasticSearch backend (2.1 minimum, 6.x maximum. Does not support 7.x+)
 * Apache HTTP Server frontend with mod_lua loaded OR
   * Nginx with nginx-extras (ng-lua module) AND lua-apr installed
 * Python 3.x for importing (with elasticsearch and formatflowed via pip)
diff --git a/docs/API.md b/docs/API.md
deleted file mode 100644
index e8eba8f..0000000
--- a/docs/API.md
+++ /dev/null
@@ -1,225 +0,0 @@
-# Pony Mail Archive API
-
-### Fetching a specific email:
-
-Usage:
-`GET /api/email.lua?id=$mid[&attachment=true&file=$hash]`
-
-Parameters: (cookie may be required)
-  - $mid: The email ID or Message-ID: header
-  - $hash: the file attachment hash
-
-Response example:
-
-~~~
-{
-    "references": "<git-pr-18-any23@git.apache.org>",
-    "from_raw": "lewismc <git@git.apache.org>",
-    "message-id": "<20150905153416.0CDCFDFE66@git1-us-west.apache.org>",
-    "@import_timestamp": "2015/10/04 09:52:41",
-    "body": "Body of email here...",
-    "request_id": "06b318af97ca96c115e878c14d0814a53407751c31388410421c1751@1441467256@<dev.any23.apache.org>",
-    "@version": 1,
-    "attachments": {},
-    "list": "<dev.any23.apache.org>",
-    "date": "2015/09/05 17:34:16",
-    "from": "lewismc <git@git.apache.org>",
-    "gravatar": "a676c0bf448fcd49f588249ead719b4c",
-    "in-reply-to": "<git-pr-18-any23@git.apache.org>",
-    "epoch": 1441467256,
-    "subject": "[GitHub] any23 pull request: Gsoc 2015 Microformats2",
-    "mid": "06b318af97ca96c115e878c14d0814a53407751c31388410421c1751@1441467256@<dev.any23.apache.org>",
-    "private": false,
-    "tid": "06b318af97ca96c115e878c14d0814a53407751c31388410421c1751@1441467256@<dev.any23.apache.org>",
-    "list_raw": "<dev.any23.apache.org>"
-}
-
-Note: date and epoch are in UTC
-
-~~~
-
-
-### Fetching list data
-Usage:
-`GET /api/stats.lua?list=$list&domain=$domain[&d=$timespan][&q=$query][&header_from=$from][&header_to=$to][&header_subject=$subject][&header_body=$body][&quick][&emailsOnly][&s=$s&e=$e]`
-
-Parameters:
-
-    - $list: The list prefix (e.g. `dev`). Wildcards may be used
-    - $domain: The list domain (e.g. `httpd.apache.org`). Wildcards may be used
-    - $timespan: A [timespan](#Timespans) value
-    - $s: yyyy-mm start of month (day 1)
-    - $e: yyyy-mm end of month (last day)
-    - $query: A search query (may contain wildcards or negations):
-      - `foo`: Find all documents containing `foo` in headers or body
-      - `-foo`: Find all documents NOT containing `foo`.
-      - `foo*`: Find all documents containing `foo`, `fooa`, `foob` etc
-    - $from: Optional From: address
-    - $to: Optional To: address
-    - $subject: Optional Subject: line
-    - $body: Optional body text
-
-Options:
-
-    - quick: only return list of email epochs
-    - emailsOnly: only return list of emails; omit thread structure, top 10 participants and word-cloud
-    
-Response example:
-
-~~~
-{
-    "took": 437179,
-    "firstYear": 2015,
-    "emails": {
-        {
-            "list_raw": "<dev.ponymail.apache.org>",
-            "gravatar": "66cf545ca7a1b8f595282bb9d8a59657",
-            "id": "b1d6446f5cc8f4846454cbabc48ddb08afbb601a77169f8e32e34102@<dev.ponymail.apache.org>",
-            "epoch": 1474883100,
-            "subject": "Re: Missing tag for 0.9 release",
-            "message-id": "<7f249f5e-e422-68a5-d57f-bfce585e638e@apache.org>",
-            "private": false,
-            "irt": "<CAOGo0VYrCOR=820LSDZA=czc==SOwCaRKasaEvVuxtUEXp9SDQ@mail.gmail.com>",
-            "from": "Daniel Gruno <h...@apache.org>",
-            "attachments": 0
-        },...
-    },
-    "no_threads": 10,
-    "domain": "ponymail.info",
-    "participants": {
-        {
-            "count": 3,
-            "name": "Daniel Gruno",
-            "gravatar": "66cf545ca7a1b8f595282bb9d8a59657",
-            "email": "hu...@apache.org"
-        }, ...
-    },
-    "lastYear": 2015,
-    "name": "dev",
-    "cloud": {...},
-    "hits": 25,
-    "thread_struct": {...},
-    thread_struct":
-    {
-        "nest": 2,
-        "children": {
-            {
-                "children": {
-                    {
-                        "children": {
-                            {
-                                "children": { },
-                                epoch: ...,
-                                tid: ...,
-                                nest: 1
-                            }
-                        },
-                        epoch: ...,
-                        tid: ...,
-                        nest: 2
-                    }
-                },
-                "epoch": 1474883100,
-                "tid": "b1d6446f5cc8f4846454cbabc48ddb08afbb601a77169f8e32e34102@<dev.ponymail.apache.org>",
-                "nest": 2
-            }
-        },
-        epoch: ...,
-        tid: ...,
-        body: ...
-    },
-    "max": 5000,
-    "searchlist": "<dev.ponymail.info>",
-    "list": "dev@ponymail.info",
-    "numparts": 0,
-    "using_wc": false
-}
-~~~
-
-### <a name="Timespans"></a>Timespans
-
-Timespans supported by the &d= parameter.
-
-    - d=yyyy-mm => equivalent to &s=yyyy-mm&e=yyyy-mm
-    - d=lte=n[wMyd] (less than n[wMyd] ago, inclusive)
-    - d=gte=n[wMyd] (more than n[wMyd] ago, inclusive)
-    - d=.*dfr=yyyy-mm-dd.* (start date for search, inclusive)
-    - d=.*dto=yyyy-mm-dd.* (end date for search, inclusive)
-    - [wMyd] = weeks, Months, years, days
-    - lte and gte are mutually exclusive
-    - dfr and dto are normally both present
-
-### Fetching preferences and quick list overview
-Usage:
-`GET /api/preferences.lua[?logout][?associate=$email][?verify&hash=$hash][?removealt=$email][?save][?addfav=$list][?remfav=$list]`
-
-Parameters: (cookie required)
-  - logout: Whether to log out of the system (optional)
-  - associate=$email - associate the account with the $email address
-  - verify&hash=$hash - verify an association request $hash
-  - removealt=$email - remove an alternate $email address
-  - save - save preferences
-  - addfav=$list - add a favourite $list
-  - remfav=$list - remove a favourite $list
-
-
-Response example:
-
-~~~
-{
-    "lists": {
-        "ponymail.info": {
-            "user": 5,
-            "dev": 36,
-            "commits": 279
-        }
-    },
-    "descriptions": {
-    },
-    "preferences": {
-        "displayMode": "threaded",
-        "hideStats": "no",
-        "theme": "default",
-        "notifications": "direct",
-        "sortOrder": "forward",
-        "compactQuotes": "yes",
-        "fullname": "Daniel Gruno",
-        "groupBy": "thread"
-    },
-    "took": 38487,
-    "login": {
-        "notifications": 0,
-        "credentials": {
-            "fullname": "Daniel Gruno",
-            "email": "foo@bar.tld"
-        }
-    }
-}
-~~~
-
-
-### Fetching notifications for a logged in user
-Usage:
-`GET /api/notifications.lua`
-
-Parameters: `None` (cookie required)
-
-
-Response example:
-
-~~~
-{
-    "notifications": {...}
-}
-~~~
-
-### Fetching a month's data as an mbox file
-Usage:
-`GET /api/mbox.lua?list=issues@ponymail.apache.org&date=2016-06`
-
-Response example:
-
-~~~
-TBA
-~~~
-
diff --git a/docs/ARCHIVING.md b/docs/ARCHIVING.md
deleted file mode 100644
index 75246ba..0000000
--- a/docs/ARCHIVING.md
+++ /dev/null
@@ -1,95 +0,0 @@
-# Archiving New Emails to Pony Mail #
-This document exists to extend the [general install guide](INSTALLING.md) provide examples on how to archive emails.
-
-__Note:__ If you plan on [importing old emails from an archive](IMPORTING.md),
-please set up the archiver __first__ so as to create an overlap of new emails
-coming in and old emails being imported. The system is designed to handle this
-without creating duplicate entries in the archive.
-
-## Mailman 2.x example:
-Set up a Pony Mail mail account/alias on a machine. This can be your local mail
-server, it can be the machine that Pony Mail is on (install sendmail or postfix
-etc there), or it can be any other machine with access to the ElasticSearch
-database that Pony Mail uses.
-
-
-### Pre-requisites
-If this is not the machine Pony Mail was installed on, you'll need to copy the
-tools/ directory from your Pony Mail installation to this machine and adjust
-ponymail.cfg to point to the right place for the database. You will also need
-Python 3 and the helper libraries installed
-(`pip3 install elasticsearch formatflowed netaddr`)
-
-### Create an alias:
-Set up a mail alias for public and private lists in `/etc/aliases` or similar method,
-and point them at the archiver script in tools/:
-
-~~~
-# You may need to add "--altheader delivered-to" to these commands, it varies
-foo-public: "|/usr/bin/python3 /path/to/tools/archiver.py"
-foo-private: "|/usr/bin/python3 /path/to/tools/archiver.py --private"
-~~~
-
-Once done, run `newaliases` to update your alias DB.
-
-### Subscribe the aliases to your mailing lists
-Use the mailman UI or CLI to subscribe foo-public@ to your public lists and
-foo-private to your private lists. Don't worry, the contents of private lists
-are hidden by default till the correct AAA scripting is set up.
-
-
-## ezmlm example:
-First, see the general introduction in the MM2 example, as this applies here as well.
-
-### Create an alias:
-Set up a dot-forward file for a public and a private alias:
-
-~~~
-.qmail-archive-public:
-    "|/usr/bin/python3 /path/to/tools/archiver.py"
-
-.qmail-archive-private:
-    "|/usr/bin/python3 /path/to/tools/archiver.py --private"
-~~~
-
-
-### Subscribe the aliases to your mailing lists
-Use the ezmlm CLI to subscribe your new aliases to the lists:
-`ezmlm-sub foolist/ archive-public@yourhost.tld`
-`ezmlm-sub secretlist/ archive-private@yourhost.tld`
-
-
-## Setting up AAA
-If you have an custom OAuth2 provider and a binary approach to private access
-(either/or), you can enable private access to people by having a key/value pair
-called `isMember` set to `true` in your JSON response from the OAuth server,
-provided it is set as an authority in config.lua. This will provide anyone
-defined as a member via OAuth full access to all private lists.
-
-If you use LDAP, you can modify the LDAP queries in the example AAA file to suit
-your organization.
-
-
-## Importing/Archiving HTML-only emails
-Should you need to import HTML-only emails into the archive, you may enable this
-with the `--html2text` command line arg. This requires that the `html2text` Python3 package
-is installed beforehand.
-
-## Munging list IDs
-If you need to rewrite list IDs on the fly as emails come in, you can use the debug.cropout 
-setting for this (in `ponymail.cfg`). 
-
-You can either use it to just crop away something:
-~~~
- [debug]
-  # Remove 'foo' from all list IDs
-  cropout:  foo
-~~~
-
- Or you can use it as a regex substitution:
-~~~
- [debug]
-  #Replace '*.bar.tld' with '*.blorg.bar.tld'
-  cropout:  <([a-z]+)\.bar\.tld> <\1.blorg.bar.tld>
-~~~
-  
diff --git a/docs/BUILDING.md b/docs/BUILDING.md
deleted file mode 100644
index 9ea97b7..0000000
--- a/docs/BUILDING.md
+++ /dev/null
@@ -1,19 +0,0 @@
-# Building Pony Mail for Production #
-Most of Pony Mail is ready-for-deployment files that just need to be checked out
-in order to work. Some areas, such as the JavaScript needs to be combined by a script,
-as they have been split into several smaller files to make it easier to find and
-work on various elements of the rendering process.
-
-### Building the JavaScript chunks ###
-All JavaScript edits should be done to the `site/js/dev/*.js` files.
-Once done, you should run combine.sh in the `site/js/dev` directory 
-to generate ponymail.js from the scripts in the dev dir:
-
-    $cd site/js/dev
-    $bash combine.sh
-    Combining JS...
-    Done!
-    $
-
-You may choose to commit the initial JS changes first before 
-committing the new combined JS, but that's up to you.
diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md
deleted file mode 100644
index 9d25f45..0000000
--- a/docs/CONTRIBUTING.md
+++ /dev/null
@@ -1,33 +0,0 @@
-
-# Contributing to Pony Mail #
-We'd LOVE it if more people would contribute to Pony Mail!
-Any form of contribution is most welcome, whether it be programming,
-documentation, evangelism, marketing, or helping out other users.
-
-Pony Mail is first and foremost a user interface, and as such, we
-are always on the lookout for user experiences. If you have used 
-Pony Mail and have feedback or ideas you wish to share, please let
-us know either through an issue/PR here or on the mailing list.
-
-## Code ##
-
-The Pony Mail code is on GitHub at https://github.com/apache/incubator-ponymail
-and pull requests are welcome.
-
-## Mailing lists ##
-
-Developers list:
- - list: dev@ponymail.incubator.apache.org
- - subscribe addr: dev-subscribe@ponymail.incubator.apache.org
- - Online version: http://lists.apache.org/list.html?dev@ponymail.incubator.apache.org
-    
-Issues list:
- - list: issues@ponymail.incubator.apache.org
- - subscribe addr: issues-subscribe@ponymail.incubator.apache.org
- - Online version: http://lists.apache.org/list.html?bugs@ponymail.incubator.apache.org
-
-## Chat ##
-    
-IRC:
-    - #ponymail on Freenode
-    
diff --git a/docs/DESIGN-NOTES.md b/docs/DESIGN-NOTES.md
deleted file mode 100644
index 1feee34..0000000
--- a/docs/DESIGN-NOTES.md
+++ /dev/null
@@ -1,76 +0,0 @@
-# Design Notes
-
-This file is an attempt to summarise some of the design issues.
-
-## Database
-The project uses the ElasticSearch (ES) database to store the mails as individual documents.
-The database stores each mail to each list as a separate document.
-If the same mail was sent to multiple lists, then it exists as multiple documents in the database.
-
-ES requires that each distinct document has a unique id (MID).
-The MID is used to insert the document in the database, and can be used to fetch it.
-
-### Database design
-The mails are stored in two separate ES indexes:
-* "mbox" - this stores information about the document, plus the parsed content, and is used for searching and summary displays.
-* "mbox_source" - this is used to store the raw content of the document.
-The two versions of the document are linked by using the same MID.
-
-### Requirements for the MID
-As mentioned above, each different document must have a unique id (MID).
-This document may arrive as a single mail message, or be loaded from a collection such as an mbox file.
-
-Duplicate database entries can be avoided by ensuring that the same MID is calculated regardless of the input source.
-[If the same message is processed more than once, it then does not matter as only the last instance will be stored.]
-The MID format does not have to be transparent; it can be an opaque hash.
-
-### Generation of the MID
-The same message may be sent to multiple lists, so the message data alone is not sufficient to identify it uniquely.
-The same message may potentially be sent more than once to the same list,
-so the combination of message and listname is also not sufficient to identify a message.
-
-Many messages will have a Message-Id header which is intended to be unique to the message.
-However this may not be the case, and some messages do not have one.
-
-Many mailing list servers will allocate a squence number or other such id to each message they send.
-This should be unique for the list, assuming that sequence is not reset.
-
-Where the Message-Id and List Server Id both exist, they can be combined to generate a MID.
-[If the List Server Id is known to be unique, then that can potentially be used alone.] 
-
-Where one or other id does not exist, then alternative means need to be used to generate the MID.
-The data used to do so must be present it all supported message sources.
-
-Algorithms for the generator remain TBA
-
-### Permalink requirements
-The application provides Permalinks which can later be used to refer to any document in the database.
-Once published, it is important that such links must continue to work.
-
-Links should be portable; i.e. if the raw messages are loaded into a new archive it should be possible
-to support existing published Permalinks.
-
-Multiple links may refer to the same document, however each link should refer to a single document.
-Ideally the Permalink should be relatively short; however that may conflict with the uniqueness requirement.
-
-It may be useful for the Permalink format to be relatively transparent.
-For example, a current ASF mod_mbox link looks like:
-
-http://mail-archives.apache.org/mod_mbox/ponymail-commits/201605.mbox/<1f73b4e0fc1a4fbbbfe4d155293c2f1a@git.apache.org>
-
-This includes a reference to the:
-- mailing list name (ponymail-commits)
-- month when mail was sent (201605.mbox)
-- the Message-Id (<1f73b4e0fc1a4fbbbfe4d155293c2f1a@git.apache.org>)
-
-This information should be sufficient to find the message in just about any mail-archive.
-
-Whereas vendor-specific links may be much shorter, but are only valid for the particular service.
-For example the equivalent Markmail link is:
-http://markmail.org/message/oanktcpxlxkmyora
-
-There may be use cases for both styles of link.
-
-### Permalink design
-TBA
-
diff --git a/docs/IMPORTING.md b/docs/IMPORTING.md
deleted file mode 100644
index 465cd28..0000000
--- a/docs/IMPORTING.md
+++ /dev/null
@@ -1,80 +0,0 @@
-# Importing Data to Pony Mail #
-Pony Mail supports many ways of importing your old mail archives via the
-`import-mbox.py` script. For command line argument tips, run `python3
-import-mbox.py --help`.
-
-Imports are digested equally every time (*), so you can
-import from the same source multiple times without creating duplicate emails in
-the archive. Both the archiver and the importer use the same digest method, so
-they can overlap. Usually, you'll want to set up the archiver first, and when
-emails start flowing through it, you'll use the importer to import older emails.
-
-## Importing from mod_mbox
-
-### Importing a single domain
-Provided you have the main mod_mbox page at https://your.tld/mod_mbox/ and your (sub)domain resources at
-https://your.tld/mod_mbox/$list-yourdomain/, you can import all lists from that domain using:
-
-`python3 import-mbox.py --source https://your.tld/mod_mbox/ --mod-mbox --project yourdomain`
-
-For a quick update, which only imports the last 2 months of mail, append the `--quick` flag.
-
-You can also import just a single list by specifying that list ID:
-
-`python3 import-mbox.py --source https://your.tld/mod_mbox/ --mod-mbox --project listname-yourdomain`
-
-### Importing an entire archive (multiple domains)
-To import an entire site, use the same command as above, but omit the `--project` flag
-
-`python3 import-mbox.py --source https://your.tld/mod_mbox/ --mod-mbox`
-
-### Setting the domain or list id properly in case of variance
-If your old archive varies in terms of list IDs across time, you can force harmonization by using the `--lid` or `--domain` flags:
-
-`python3 import-mbox.py --source https://your.tld/mod_mbox/ --mod-mbox --project listid-yourdomain --lid "<listid.yourdomain.tld>"`
-
-This should only be done one list at a time.
-
-## Importing from Pipermail
-To import from pipermail, you will have to run the import one list at a time. As with mod_mbox imports, you must specify a source, but use `--pipermail` instead of `--mod-mbox`:
-
-`python3 import-mbox.py --source https://your.tld/pipermail/foolist/ --pipermail`
-
-### Pipermail and html-only emails
-While you can convert HTML-only emails to text using `--html2text`, Pipermail has some peculiarities
-where it adds a text/plain message to these emails, thus preventing html2text from working. You can
-circumvent this by using the `--ignorebody "foo"` arg to ignore all text/plain bodies containing `foo`.
-
-While the `project` flag is not needed here, you may wish to specify the list ID for the import.
-
-## Importing from locally stored mbox files
-To import from one or more local mbox files, specify a filesystem path as the source:
-
-`python3 import-mbox.py --source /tmp/mylists/`
-
-This will recursively import all files with the extension '.mbox'.
-
-You can change the extension as follows:
-
-`python3 import-mbox.py --source /tmp/mylists/ --ext .mbx`
-
-To match all files with any non-empty extension:
-
-`python3 import-mbox.py --source /tmp/mylists/ --ext '.*'`
-
-To match files regardless of extension:
-
-`python3 import-mbox.py --source /tmp/mylists/ --ext ''`
-
-Or you can import a single file:
-
-`python3 import-mbox.py --source 2016-11.mbox`
-
-(This is supported in versions after 0.9)
-
-## Test archives
-We have a few test archives for those that wish to test large imports.
-They can be found in gzip format at [http://ponymail.info/mboxes/](http://ponymail.info/mboxes/)
-
-(*) The digest depends on the [archiver] generator setting in ponymail.cfg
-If that varies between imports, then duplicates will occur
diff --git a/docs/INSTALL.centos.md b/docs/INSTALL.centos.md
deleted file mode 100644
index 99e2882..0000000
--- a/docs/INSTALL.centos.md
+++ /dev/null
@@ -1,141 +0,0 @@
-# Installing Pony Mail on CentOS 7.1: #
-This installation is a bit trickier, as CentOS does not have
-Python 3 or any of the lua modules in its default package system.
-
-Start by installing the following CentOS packages:
-
-- httpd
-- git
-- lua
-- lua-devel
-- gcc
-- gcc-c++
-- kernel-dev
-- unzip
-- openssl
-- openssl-devel
-- readline-devel
-
-~~~
-sudo yum install -y httpd git lua lua-devel gcc gcc-c++ kernel-devel unzip openssl openssl-devel readline-devel
-~~~
-
-
-Then, proceed to build LuaRocks (for lua deps):
-
-~~~
-wget http://luarocks.org/releases/luarocks-2.0.6.tar.gz    
-tar zxvf luarocks-2.0.6.tar.gz                                             
-cd luarocks-2.0.6                                                               
-./configure                                                                          
-make                                                                                  
-sudo make install
-~~~
-
-Now build/install the required Lua modules:
-
-~~~
-sudo luarocks install lua-socket
-sudo luarocks install luasec OPENSSL_LIBDIR=/usr/lib64/
-sudo luarocks install lua-cjson
-~~~
-
-
-Configure, compile and install Python 3:
-
-~~~
-sudo yum groupinstall -y development
-sudo yum install -y zlib-dev sqlite-devel bzip2-devel xz-libs
-wget http://www.python.org/ftp/python/3.4.3/Python-3.4.3.tar.xz
-xz -d Python-3.4.3.tar.xz
-tar zvf Python-3.4.3.tar
-cd Python-3.4.3/
-./configure
-make
-sudo make altinstall
-~~~
-
-
-Install the required Python 3 modules:
-~~~
-sudo pip3.4 install elasticsearch formatflowed chardet netaddr
-~~~
-
-
-Install ElasticSearch:
-
-~~~
-sudo yum install -y java-1.7.0-openjdk-headless
-sudo rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch
-
-    (The following is taken from the ElasticSearch online guide:)
-
-    Add the following in your /etc/yum.repos.d/ directory in a file with a .repo suffix,
-    for example elasticsearch.repo:
-    
-    [elasticsearch-1.7]
-    name=Elasticsearch repository for 1.7.x packages
-    baseurl=http://packages.elastic.co/elasticsearch/1.7/centos
-    gpgcheck=1
-    gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch
-    enabled=1
-
-sudo yum update
-sudo yum install elasticsearch
-~~~
-
-
-Configure and start up ElasticSearch:
-
-~~~
-sudo sudo /bin/systemctl daemon-reload
-sudo sudo /bin/systemctl enable elasticsearch.service
-sudo /etc/init.d/elasticsearch start
-~~~
-
-
-Check out a copy of Pony Mail:
-~~~
-cd /var/www
-git clone https://github.com/apache/incubator-ponymail.git
-~~~
-
-
-Set up Pony Mail:
-~~~
-cd /var/www/ponymail/tools
-python3.4 setup.py
-[... answer questions asked by the setup script ...]
-~~~
-
-
-Set up Apache httpd by adding, for example, the following virtual host configuration:
-This differs from the normal installation (because of CentOS specifics), so beware
-
-~~~
-<VirtualHost *:80>
-    LuaPackageCPath /usr/local/lib/lua/5.1/?.so
-    LuaPackagePath  /usr/local/share/lua/5.1/?.lua
-    ServerName mylists.foo.tld
-    DocumentRoot /var/www/ponymail/site
-    AddHandler      lua-script .lua
-    LuaScope        thread
-    LuaCodeCache    stat
-    AcceptPathInfo  On
-</VirtualHost>
-~~~
-
-(re)start apache:
-
-~~~
-sudo apachectl restart
-~~~
-
-Once this is done, you should now have a *working copy* of Pony Mail!
-
-You may wish to tweak the settings in `site/js/config.js` and your
-elasticsearch settings once Pony mail is up and running.
-
-Refer to the [General installation documentation](INSTALLING.md) for
-detailed information about archiving messages, OAuth, mail settings and
-much more.
diff --git a/docs/INSTALL.debian.md b/docs/INSTALL.debian.md
deleted file mode 100644
index 33b058d..0000000
--- a/docs/INSTALL.debian.md
+++ /dev/null
@@ -1,78 +0,0 @@
-# Installing Pony Mail on Debian Jessie: #
-Start by installing the following Debian packages:
-
-- apache2
-- git
-- lua-sec
-- lua-cjson
-- lua-socket
-- python3
-- python3-pip
-
-~~~
-sudo apt-get install apache2 git lua-cjson lua-sec lua-socket python3 python3-pip
-~~~
-
-Install the required Python 3 modules:
-~~~
-sudo pip3 install elasticsearch formatflowed netaddr
-~~~
-
-Install ElasticSearch:
-
-~~~
-sudo apt-get install openjdk-7-jre-headless
-wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
-echo "deb http://packages.elastic.co/elasticsearch/1.7/debian stable main" | sudo tee -a /etc/apt/sources.list.d/elasticsearch-1.7.list
-sudo apt-get update && sudo apt-get install elasticsearch
-~~~
-
-
-Check out a copy of Pony Mail:
-~~~
-cd /var/www
-sudo git clone https://github.com/apache/incubator-ponymail.git
-~~~
-
-Start up ElasticSearch:
-
-~~~
-sudo service elasticsearch start
-~~~
-
-Set up Pony Mail:
-~~~
-cd /var/www/ponymail/tools
-sudo python3 setup.py
-[... answer questions asked by the setup script ...]
-~~~
-
-
-Set up Apache httpd by adding, for example, the following virtual host configuration:
-
-~~~
-<VirtualHost *:80>
-    ServerName mylists.foo.tld
-    DocumentRoot /var/www/ponymail/site
-    AddHandler      lua-script .lua
-    LuaScope        thread
-    LuaCodeCache    stat
-    AcceptPathInfo  On
-</VirtualHost>
-~~~
-
-Enable mod_lua and start apache:
-
-~~~
-sudo a2enmod lua
-sudo service apache start
-~~~
-
-Once this is done, you should now have a *working copy* of Pony Mail!
-
-You may wish to tweak the settings in `site/js/config.js` and your
-elasticsearch settings once Pony mail is up and running.
-
-Refer to the [General installation documentation](INSTALLING.md) for
-detailed information about archiving messages, OAuth, mail settings and
-much more.
diff --git a/docs/INSTALL.fedora.md b/docs/INSTALL.fedora.md
deleted file mode 100644
index f27e1ee..0000000
--- a/docs/INSTALL.fedora.md
+++ /dev/null
@@ -1,112 +0,0 @@
-# Installing Pony Mail on Fedora 22: #
-
-Start by installing the following Fedora packages:
-
-- httpd
-- git
-- lua
-- lua-sec
-- lua-socket
-- python 3
-- luarocks
-
-~~~
-sudo dnf install -y httpd git lua lua-sec lua-socket python3 luarocks
-~~~
-
-Install the missing cjson package via luarocks:
-
-~~~
-sudo luarocks-5.3 install lua-cjson
-~~~
-
-Install the required Python 3 modules:
-~~~
-sudo pip3.4 install elasticsearch formatflowed chardet netaddr
-~~~
-
-
-Install ElasticSearch:
-
-~~~
-sudo dnf install -y java-1.8.0-openjdk-headless
-sudo rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch
-
-    (The following is taken from the ElasticSearch online guide:)
-
-    Add the following in your /etc/yum.repos.d/ directory in a file with a .repo suffix,
-    for example elasticsearch.repo:
-    
-    [elasticsearch-1.7]
-    name=Elasticsearch repository for 1.7.x packages
-    baseurl=http://packages.elastic.co/elasticsearch/1.7/centos
-    gpgcheck=1
-    gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch
-    enabled=1
-
-
-sudo dnf install -y elasticsearch
-~~~
-
-
-Configure and start up ElasticSearch:
-
-~~~
-sudo /bin/systemctl daemon-reload
-sudo /bin/systemctl enable elasticsearch.service
-sudo /etc/init.d/elasticsearch start
-~~~
-
-
-Check out a copy of Pony Mail:
-~~~
-cd /var/www
-sudo git clone https://github.com/apache/incubator-ponymail.git
-~~~
-
-
-Set up Pony Mail:
-~~~
-cd /var/www/ponymail/tools
-sudo python3.4 setup.py
-[... answer questions asked by the setup script ...]
-~~~
-
-
-Set up Apache httpd by adding, for example, the following virtual host configuration:
-This differs from the normal installation (because of CentOS specifics), so beware
-
-~~~
-<VirtualHost *:80>
-    LuaPackageCPath /usr/lib/lua/5.3/?.so
-    LuaPackagePath  /usr/share/lua/5.3/?.lua
-    ServerName mylists.foo.tld
-    DocumentRoot /var/www/ponymail/site
-    AddHandler      lua-script .lua
-    LuaScope        thread
-    LuaCodeCache    stat
-    AcceptPathInfo  On
-</VirtualHost>
-~~~
-
-(re)start apache:
-
-~~~
-sudo apachectl restart
-~~~
-
-IF you have SELinux running, you need to allow httpd (apache) to
-be able to connect to remotes, otherwise Pony Mail won't work:
-
-~~~
-sudo setsebool -P httpd_can_network_connect 1
-~~~
-
-Once this is done, you should now have a *working copy* of Pony Mail!
-
-You may wish to tweak the settings in `site/js/config.js` and your
-elasticsearch settings once Pony mail is up and running.
-
-Refer to the [General installation documentation](INSTALLING.md) for
-detailed information about archiving messages, OAuth, mail settings and
-much more.
diff --git a/docs/INSTALL.ubuntu.md b/docs/INSTALL.ubuntu.md
deleted file mode 100644
index 95c47a2..0000000
--- a/docs/INSTALL.ubuntu.md
+++ /dev/null
@@ -1,94 +0,0 @@
-# Installing Pony Mail on Ubuntu 14.04 or 16.04: #
-Start by installing the following Ubuntu packages:
-
-- apache2
-- git
-- liblua5.2-dev
-- lua-sec
-- lua-cjson
-- lua-socket
-- python3
-- python3-pip
-- subversion
-
-~~~
-sudo apt-get install apache2 git liblua5.2-dev lua-cjson lua-sec lua-socket python3 python3-pip subversion
-~~~
-
-Install the required Python 3 modules:
-~~~
-sudo pip3 install elasticsearch formatflowed netaddr
-~~~
-
-Install ElasticSearch:
-~~~
-wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
-echo "deb http://packages.elastic.co/elasticsearch/2.x/debian stable main" | sudo tee -a /etc/apt/sources.list.d/elasticsearch-2.x.list
-sudo apt-get update && sudo apt-get install elasticsearch default-jre-headless
-~~~
-
-Compile and install mod_lua if necessary (httpd < 2.4.17 on Ubuntu):
-~~~
-sudo apt-get install apache2-dev
-svn co https://svn.apache.org/repos/asf/httpd/httpd/branches/2.4.x/modules/lua/
-cd lua/
-sudo apxs -I/usr/include/lua5.2 -cia mod_lua.c lua_*.c -lm -llua5.2
-~~~
-
-Check out a copy of Pony Mail:
-~~~
-sudo git clone https://github.com/apache/incubator-ponymail.git /var/www/ponymail
-~~~
-
-Configure Elasticsearch to automatically start during bootup. For Ubuntu <= 14.10:
-~~~
-sudo update-rc.d elasticsearch defaults 95 10
-~~~
-
-For Ubuntu >= 15.04:
-~~~
-sudo /bin/systemctl daemon-reload
-sudo /bin/systemctl enable elasticsearch.service
-~~~
-
-Start up ElasticSearch:
-
-~~~
-sudo service elasticsearch start
-~~~
-
-Set up Pony Mail:
-~~~
-cd /var/www/ponymail/tools
-sudo python3 setup.py
-[... answer questions asked by the setup script ...]
-~~~
-
-
-Set up Apache httpd by adding, for example, the following virtual host configuration (e.g. in `/etc/apache2/sites-enabled/000-default.conf`):
-~~~
-<VirtualHost *:80>
-    ServerName mylists.foo.tld
-    DocumentRoot /var/www/ponymail/site
-    AddHandler      lua-script .lua
-    LuaScope        thread
-    LuaCodeCache    stat
-    AcceptPathInfo  On
-</VirtualHost>
-~~~
-
-Enable mod_lua and start apache, if not already enabled:
-
-~~~
-sudo a2enmod lua
-sudo service apache start
-~~~
-
-Once this is done, you should now have a *working copy* of Pony Mail!
-
-You may wish to tweak the settings in `site/js/config.js` and your
-elasticsearch settings once Pony mail is up and running.
-
-Refer to the [General installation documentation](INSTALLING.md) for
-detailed information about archiving messages, OAuth, mail settings and
-much more.
diff --git a/docs/INSTALLING.md b/docs/INSTALLING.md
deleted file mode 100644
index 3f3fb4f..0000000
--- a/docs/INSTALLING.md
+++ /dev/null
@@ -1,215 +0,0 @@
-# Installing Pony Mail #
-
-
-If your distro is on this list, please refer to that specific document
-for detailed package installation instructions:
-
-- [Debian (Jessie) Installation Instructions](INSTALL.debian.md)
-- [Ubuntu (14.04) Installation Instructions](INSTALL.ubuntu.md)
-- [CentOS (7.1) Installation Instructions](INSTALL.centos.md)
-- [Fedora (22) Installation Instructions](INSTALL.fedora.md)
-
-Otherwise, read the next two chapters:
-
-## Pre-requisites ##
-You will need the following software installed on your machine:
-
-- ElasticSearch >= 2.1
-- Python 3.x for the archiver plugin (setup.py will handle dependencies) and importer
-- Python `html2text` package (GPLv3) if you wish to archive HTML-only mails (remember to add the `--html2text` command line arg)
-- Apache HTTP Server 2.4.x with mod_lua (see http://modlua.org/gs/installing if you need to build mod_lua manually)
-- Lua >=5.1 with the following modules: cjson, luasec, luasocket
-  (Note: Lua 5.3 is not currently supported by httpd mod_lua or luasocket)
-
-
-## Download and Install ##
-
-- Download the git repo: `git clone https://github.com/apache/incubator-ponymail.git`
-- Start ElasticSearch on the machine it needs to run on.
-- Run setup.py in the `tools` dir:
-```
-      $cd tools
-      $python3 setup.py
-      ...[follow instructions in the setup script]
-```
-- Edit `site/js/config.js` to suit your needs (usually very little editing is needed)
-
-
-
-### Using auth for ElasticSearch ###
-If your ElasticSearch instance requires authentication for the importer/archiver, please
-add the following lines in the `elasticsearch` block of `ponymail.cfg` once generated:
-
-~~~
-user:           [username for ES]
-password:       [password for ES]
-~~~
-
-
-### Using Apache HTTP Server: ###
-- Set up a VirtualHost block in Apache httpd that points to the `site/` directory in Pony Mail
-- Add the configuration snippets from `configs/ponymail_httpd.conf` to the virtual host
-- Start Apache httpd to enable the user-facing interface
-
-### Using nginx: ###
-- To use nginx, you will also need to install the `lua-apr` module from your distro.
-- Set up a Server block in nginx that points to the `site/` directory in Pony Mail
-- Add the configuration snippets from `configs/ponymail_nginx.conf` to the server config
-- Start nginx to enable the user-facing interface
-
-
-## Setting up the archiver ##
-First off, you will need both `tools/archiver.py` and the generated
-`tools/ponymail.cfg` present on the machine that your mail server runs on. This
-machine should also have access to the ElasticSearch backend.
-
-If your mailing list supports feeding emails to a program, feed the incoming new
-emails to `python3 /path/to/tools/archiver.py` and it will use STDIN as the
-transport mechanism. If you are simply using aliases or dot-forwards and no ML
-system, you can add (for example) `"|/usr/bin/python3
-/path/to/tools/archiver.py"` to your alias file to enable archiving. If you are
-not using a Mailing List manager, you will need to tell Pony Mail which email
-header determines the list ID using the --altheader argument, for instance:
-```
-    foolist: "|/usr/bin/python3 /path/to/tools/archiver.py --altheader delivered-to"
-    foolist-private: "|/usr/bin/python3 /path/to/tools/archiver.py --altheader delivered-to --private"
-```
-
-If you are using MailMan 3, you can add archiver.py as an archive by following the instructions inside the python script:
-- Copy the archiver.py file to `$mailman_plugin_dir/mailman_ponymail/__init__.py`
-- Copy ponymail.cfg to the same dir (for ES configuration)
-- Enable the module by adding the following to your `mailman.cfg` file::
-```
-  [archiver.ponymail]
-  # Pony Mail
-  class: mailman_ponymail.Archiver
-  enable: yes
-```
-
-For older mailing list systems such as Mailman 2 and ezmlm, you can also
-tak a look at our [archiving examples](ARCHIVING.md) page for pointers.
-
-## Public versus private lists ##
-In MailMan 3, this should be auto-detected and is not a concern.
-When using other ML systems via piping to STDIN, you should add
-the --private arg to the python script to mark an email as private:
-```
-    foolist-private: "|/usr/bin/python3 /path/to/tools/archiver.py --private"
-    foolist-public: "|/usr/bin/python3 /path/to/tools/archiver.py"
-```
-
-## Importing old data into Pony Mail
-See [this guide](IMPORTING.md) for details on how to import old archives into Pony Mail.
-
-## Bulk editing lists ##
-You can use `edit-list.py` to perform bulk operations:
-- Rename lists
-- Mark entire lists are private or public
-
-Run `python3 edit-list.py --help` for CLI args.
-
-
-## Setting up OAuth for Pony Mail ##
-If you want people to be able to log in and reply via the Web UI, you
-can specify an OAuth provider.
-
-
-### Setting up an OAuth provider ###
-Pony Mail comes with a few default OAuth examples in `site/js/config.js`, such
-as ASF Oauth and Google OAuth. You can enable these by uncommenting the lines in
-question, or set up your own OAuth portal to handle things. This is a standard
-OAuth that expects the backend to supply the following JSON data on success:
-
-~~~
-    {
-        "fullname": "The full name of the authed user",
-        "email": "The user's email address",
-        "uid": "(optional) The unique user ID of the logged in user (for instance, LDAP UID)",
-        "isMember": true/false (optional, specifies whether the person is a privileged user with access to all lists)
-    }
-~~~
-
-For private list browsing, Pony Mail supplies an example AAA library in
-`site/api/lib/aaa.lua` that does LDAP lookups to determine which groups a person
-belongs to, and thus which lists said person has access to. The AAA example is
-modelled on the Apache LDAP structure, so you may wish to change this to suit
-your need. We have [several simple AAA examples](../aaa_examples/) in the
-`aaa_examples` directory.
-
-If you are looking for an OAuth portal to provide users access to private lists
-in the archive, you will need to add the OAuth domain to config.admin_oauth in
-config.lua:
-
-~~~
-    admin_oauth = { 'myoauth.foo.tld', '*.oauthprovider.com', 'etc' }
-~~~~
-
-If not specified in config.lua, OAuth will only provide users with a place to
-store settings and notifications, and - provided your mail server is set to accept
-this - a place to reply to emails in the archive.
-
-
-#### Using GitHub OAuth and other client-secret providers ####
-If your OAuth provider requires a client secret, you can specify this in `site/api/lib/config.lua`, as this GitHub example shows:
-
-~~~
-    oauth_fields = {
-        github = {
-            client_secret = "abcdef1",
-            client_id = "abcdef2",
-            oauth_token = "https://github.com/login/oauth/access_token"
-        }
-    }
-~~~
-
-This essentially overrides `config.js` but without showing the data to anyone outside the server.
-
-### Whitelisting replies via the Web UI ###
-To have Pony Mail accept replies done via the Web UI, you must make sure
-that `site/api/lib/config.lua` contains the appropriate string (or array of strings) matching the domain(s) you wish to allow new email for. To allow replies to everything, set this to `* `(NOT RECOMMENDED).
-You can also allow based on GLOBs or an array of accepted domains and sub-domains:
-
-~~~
-    accepted_domains = "*" -- This would allow posts to any email address, baaaad choice.
-    accepted_domains = "foo.org" -- Allow only to *@foo.org
-    accepted_domains = "*.foo.org" -- Allow only posts to *@*.foo.org, but not *@foo.org
-    accepted_domains = { "foo.org", "*.foo.org" } -- Allow posts both to *.foo.org and foo.org
-~~~
-
-
-### Setting email footers ###
-It is possible to set email footers in each email sent via the Web UI.
-This is done by configuring the `email_footer` variable in  `site/api/lib/config.lua`.
-You may use the following variables in the footer:
-
-~~~
-    $list: The mailing list being sent to (foo@bar.tld)
-    $hostname: The hostname of the server
-    $port: The port of the server (80, 443 etc)
-    $msgid: The message ID of the email (for permalinks etc)
-~~~
-
-An example footer could be:
-
-~~~
-    --------
-    Sent via Pony Mail for $list.
-    To view this list online, visit: https://my.tld/list.html?$list
-    To view this email (and subsequent replies), visit:
-    https://my.tld/thread.html/$msgid
-    --------
-~~~
-
-
-### A note on email headers ###
-By default, headers such as to/cc are not shown in the normal email view.
-To enable these headers, set `full_headers` to `true` in the `site/api/lib/config.lua` file.
-
-### Lastly, a note about Message-ID (MID) generators
-The default MID generator is called 'medium' and digests the message
-body, timestamp and list-ID to generate the MID. There is also a 'short'
-that only digests the body, and a 'full' that uses the entire message as
-a bytestring to generate an ID. Medium is recommended for most setups
-(especially clustered setups), while full can be used for single-machine
-setups.
-N.B. At present, all the generators have issues, see (#176 #177 #178)
diff --git a/docs/README b/docs/README
deleted file mode 100644
index b5eeff9..0000000
--- a/docs/README
+++ /dev/null
@@ -1,14 +0,0 @@
-The documentation appears in the following files:
-
-API reference: API.md
-Archiving email into Pony Mail: ARCHIVING.md
-Building Pony Mail for Production: BUILDING.md
-Contributing to Pony Mail: CONTRIBUTING.md
-Imorting existing data into Pony Mail: IMPORTING.md
-Installing:
-    CentOS (and RHEL): INSTALL.centos.md
-    Debian: INSTALL.debian.md
-    Fedora: INSTALL.fedora.md
-    Ubuntu: INSTALL.ubuntu.md
-    Anywhere else: INSTALLING.md
-
diff --git a/requirements.txt b/requirements.txt
index 49fa8c3..efa8e24 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,9 +1,8 @@
-# for pytest/unit tests
-PyYAML~=5.3.1
-elasticsearch-dsl>=5.0.0
-elasticsearch~=5.0.0
-certifi~=2020.6.20
-chardet~=3.0.4
-netaddr~=0.8.0
-formatflowed~=2.0.0
-html2text~=2019.8.11
+# The following modules are required.
+# It is believed that they have licences as shown
+# N.B. modules with license that are compatible with AL 2.0 should be included here
+# As such, html2text (GPL) and chardet (GPL) cannot be included
+certifi # MPL 2.0
+elasticsearch < 6.0.0 # AL2.0
+formatflowed # Python Software Foundation
+netaddr # BSD License, MIT License
diff --git a/site/api/lib/config.lua.sample b/site/api/lib/config.lua.sample
index 7a84a68..0acb615 100644
--- a/site/api/lib/config.lua.sample
+++ b/site/api/lib/config.lua.sample
@@ -37,6 +37,7 @@
 --  no_association = {}, -- domains that are not allowed for email association
 --  listsDisplay = 'regex', -- if defined, hide list names that don't match the regex
 --  debug = false, -- whether to return debug information
-    antispam = true  -- Whether or not to add anti-spam measures aimed at anonymous users.
+    antispam = true,  -- Whether or not to add anti-spam measures aimed at anonymous users.
+    noShowQuery = false, -- disallow return query in JSON result: true|false (default false)
 }
 return config
diff --git a/site/api/lib/elastic.lua b/site/api/lib/elastic.lua
index 4eacf5b..192cac6 100644
--- a/site/api/lib/elastic.lua
+++ b/site/api/lib/elastic.lua
@@ -62,8 +62,6 @@
     local result, hc = http.request(url, js)
     checkReturn(hc, ok404)
     local json = JSON.decode(result)
-    -- TODO should we return the http status code?
-    -- This might be necessary if codes such as 404 did not cause an error
     return json, hc
 end
 
diff --git a/site/api/stats.lua b/site/api/stats.lua
index 3e82bdb..a6af005 100644
--- a/site/api/stats.lua
+++ b/site/api/stats.lua
@@ -25,6 +25,19 @@
 local cross = require 'lib/cross'
 local utils = require 'lib/utils'
 
+-- accumulate the monthly stats
+local function monthly_emails(buckets, target)
+    for _,v in pairs(buckets) do
+        if v.doc_count > 0 then
+            if target[v.key_as_string] then
+                target[v.key_as_string] = target[v.key_as_string] + v.doc_count
+            else
+                target[v.key_as_string] = v.doc_count
+            end
+        end
+    end
+end
+
 local BODY_MAXLEN = config.stats_maxBody or 200
 -- words to exclude from word cloud:
 local EXCLUDE = config.stats_wordExclude or ".|..|..."
@@ -429,6 +442,13 @@
                                     max = {
                                         field = "epoch"
                                     }
+                                },
+                                monthly_emails = {
+                                    date_histogram = {
+                                        field = "date",
+                                        interval = "month",
+                                        format = "yyyy-MM"
+                                    }
                                 }
                             }
                         }
@@ -439,10 +459,12 @@
         datespan = {}
         datespan.pubfirst = nil
         datespan.publast = nil
+        datespan.monthly_emails = {}
         -- find public min and max (buckets will be empty if there are no matching lists)
         for _, list in pairs(doc.aggregations.lists.buckets) do
             for _, private in pairs(list.private.buckets) do
                 if private.key_as_string == "false" then
+                    monthly_emails(private.monthly_emails.buckets, datespan.monthly_emails)
                     if (datespan.publast == nil) or (private.last.value > datespan.publast) then datespan.publast = private.last.value end
                     if (datespan.pubfirst == nil) or (private.first.value < datespan.pubfirst) then datespan.pubfirst = private.first.value end
                 end
@@ -459,16 +481,16 @@
         for _, list in pairs(doc.aggregations.lists.buckets) do
             for _, private in pairs(list.private.buckets) do
                 if private.key_as_string == "true" then
+                    datespan.private = datespan.private or {}
+                    datespan.private[list.key] = datespan.private[list.key] or {}
+                    datespan.private[list.key].monthly_emails = datespan.private[list.key].monthly_emails or {}
+                    monthly_emails(private.monthly_emails.buckets, datespan.private[list.key].monthly_emails)
                     local prvlast = private.last.value
                     if prvlast > datespan.publast then
-                        datespan.private = datespan.private or {}
-                        datespan.private[list.key] = datespan.private[list.key] or {}
                         datespan.private[list.key].last = prvlast
                     end
                     local prvfirst = private.first.value
                     if prvfirst < datespan.pubfirst then
-                        datespan.private = datespan.private or {}
-                        datespan.private[list.key] = datespan.private[list.key] or {}
                         datespan.private[list.key].first = prvfirst
                     end
                 end
@@ -484,8 +506,16 @@
     local last = datespan.publast
     for lid, prvdates in pairs(datespan.private or {}) do
         if aaa.canAccessList(r, lid, account) then
-           if prvdates.first and prvdates.first < first then first = prvdates.first end
-           if prvdates.last and prvdates.last > last then last = prvdates.last end
+            -- merge the stats from the private list
+            for k,v in pairs(prvdates.monthly_emails) do
+                if datespan.monthly_emails[k] then
+                    datespan.monthly_emails[k] = datespan.monthly_emails[k] + v
+                else
+                    datespan.monthly_emails[k] = v
+                end
+            end
+            if prvdates.first and prvdates.first < first then first = prvdates.first end
+            if prvdates.last and prvdates.last > last then last = prvdates.last end
         end
     end
 
@@ -738,6 +768,7 @@
     listdata.lastYear = datespan.lastYear
     listdata.firstMonth = datespan.firstMonth
     listdata.lastMonth = datespan.lastMonth
+    listdata.monthly_emails = datespan.monthly_emails
     listdata.list = listraw:gsub("^([^.]+)%.", "%1@"):gsub("[<>]+", "")
     listdata.emails = emls
     listdata.hits = #emls
@@ -750,6 +781,10 @@
     end
     listdata.numparts = allparts
     listdata.unixtime = os.time()
+    if get.showQuery and not config.noShowQuery then
+        listdata.squery = squery
+        listdata.sdata = get
+    end
     
     r:puts(JSON.encode(listdata))
     
diff --git a/site/api/thread.lua b/site/api/thread.lua
index 741ac28..1fc0f9b 100644
--- a/site/api/thread.lua
+++ b/site/api/thread.lua
@@ -84,9 +84,6 @@
     -- Try searching by mid if not found, for backward compat
     if not doc or not doc.mid then
         local docs = elastic.find("message-id:\"" .. r:escape(eid) .. "\"", 1, "mbox")
-        if #docs == 1 then
-            doc = docs[1]
-        end
         if #docs == 0 and #eid == utils.SHORTENED_LINK_LEN then
             docs = elastic.find("mid:" .. r:escape(eid) .. "*", 1, "mbox")
         end
diff --git a/site/js/dev/ponymail_assign_vars.js b/site/js/dev/ponymail_assign_vars.js
index f9252d3..058c950 100644
--- a/site/js/dev/ponymail_assign_vars.js
+++ b/site/js/dev/ponymail_assign_vars.js
@@ -20,7 +20,7 @@
 // Thus lightening the load on the backend (caching and such)
 
 var _VERSION_ = "0.12-SNAPSHOT" // Current version (as far as we know)
-var months = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']
+var months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
 var d_ppp = 15; // results per page
 var c_page = 0; // current page position for list view
 var open_emails = [] // cache index for loaded emails
diff --git a/site/js/dev/ponymail_pagebuilder.js b/site/js/dev/ponymail_pagebuilder.js
index 8619aeb..1d6d30b 100644
--- a/site/js/dev/ponymail_pagebuilder.js
+++ b/site/js/dev/ponymail_pagebuilder.js
@@ -37,6 +37,7 @@
     var lastYear  = json.lastYear
     var firstMonth0 = json.firstMonth - 1 // 0-based
     var lastMonth0 = json.lastMonth - 1 // 0-based
+    var monthly_emails = json.monthly_emails
     
     // Build the main calendar (desktop version)
     var dp = document.getElementById('datepicker')
@@ -54,21 +55,23 @@
         if (fyear == firstYear) {
             n = "block"
         }
-        dp.innerHTML += "<label onmouseout='this.setAttribute(\"class\", \"label label-success\");'  onmouseover='this.setAttribute(\"class\", \"label label-warning\");' onclick='toggleCalendar(" + year + ");' class='label label-success' style='float: left; width: 110px; font-size: 11pt; cursor: pointer'>" + year + "</label><br/>"
-        var cale = "<div style='float: left; width: 80%; display: " + n + "; padding-left: 15px; margin-bottom: 15px;' id='cal_" + year + "'>"
+        var cale = ''
         var em = (new Date().getFullYear() == year) ? new Date().getMonth() : 11;
         for (var y = em; y >= 0; y--) {
-            var url = "list.html?" + xlist + ":" + (year+"-"+(y+1))
-            var pfx = ''
-            var sfx = ''
-            if ((year == firstYear && y < firstMonth0) || (year == lastYear && y > lastMonth0)) {
-                pfx = '<i>'
-                sfx = '</i>'
+            var yyyymm = (year+"-"+(y+1))
+            var url = "list.html?" + xlist + ":" + yyyymm
+            var yyyy0m = y < 9 ? (year+"-0"+(y+1)) : yyyymm
+            if (monthly_emails[yyyy0m]) {
+                var count = monthly_emails[yyyy0m]
+                cale += "<a href='" + url + "' onclick='return false;'><label id='calmonth_" + yyyymm + "' style='width: 80px; float: left;cursor: pointer;' class='label label-default label-hover' onclick='toggleEmail(" + year + ", " + (y + 1) + ");' >" + months[y] + ' ('+ count + ')' + "</label></a><br/>"
             }
-            cale += "<a href='" + url + "' onclick='return false;'><label id='calmonth_" + (year+"-"+(y+1)) + "' style='width: 80px; float: left;cursor: pointer;' class='label label-default label-hover' onclick='toggleEmail(" + year + ", " + (y + 1) + ");' >" + pfx + months[y] + sfx + "</label></a><br/>"
         }
-        cale += "</div>"
-        dp.innerHTML += cale
+        if (cale != '') {
+            cale += "</div>"
+            dp.innerHTML += "<label onmouseout='this.setAttribute(\"class\", \"label label-success\");'  onmouseover='this.setAttribute(\"class\", \"label label-warning\");' onclick='toggleCalendar(" + year + ");' class='label label-success' style='float: left; width: 110px; font-size: 11pt; cursor: pointer'>" + year + "</label><br/>"
+            var calehdr = "<div style='float: left; width: 80%; display: " + n + "; padding-left: 15px; margin-bottom: 15px;' id='cal_" + year + "'>"
+            dp.innerHTML += calehdr + cale
+        }
     }
     
     // Build the mobile version (dropdown)
diff --git a/site/js/ponymail.js b/site/js/ponymail.js
index 3c7e9d9..d814bc2 100644
--- a/site/js/ponymail.js
+++ b/site/js/ponymail.js
@@ -28,7 +28,7 @@
 // Thus lightening the load on the backend (caching and such)
 
 var _VERSION_ = "0.12-SNAPSHOT" // Current version (as far as we know)
-var months = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']
+var months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
 var d_ppp = 15; // results per page
 var c_page = 0; // current page position for list view
 var open_emails = [] // cache index for loaded emails
@@ -2323,7 +2323,7 @@
 // ML address: only accept valid mailing list name, domain or both
 // return true if the address is valid
 function valid_address(val) {
-    return val.match(/^[-@A-Za-z.0-9]+$/);
+    return val.match(/^[-_@A-Za-z.0-9]+$/);
 }
 
 // Check for slow URLs every 0.1 seconds
@@ -3688,6 +3688,7 @@
     var lastYear  = json.lastYear
     var firstMonth0 = json.firstMonth - 1 // 0-based
     var lastMonth0 = json.lastMonth - 1 // 0-based
+    var monthly_emails = json.monthly_emails
     
     // Build the main calendar (desktop version)
     var dp = document.getElementById('datepicker')
@@ -3705,21 +3706,23 @@
         if (fyear == firstYear) {
             n = "block"
         }
-        dp.innerHTML += "<label onmouseout='this.setAttribute(\"class\", \"label label-success\");'  onmouseover='this.setAttribute(\"class\", \"label label-warning\");' onclick='toggleCalendar(" + year + ");' class='label label-success' style='float: left; width: 110px; font-size: 11pt; cursor: pointer'>" + year + "</label><br/>"
-        var cale = "<div style='float: left; width: 80%; display: " + n + "; padding-left: 15px; margin-bottom: 15px;' id='cal_" + year + "'>"
+        var cale = ''
         var em = (new Date().getFullYear() == year) ? new Date().getMonth() : 11;
         for (var y = em; y >= 0; y--) {
-            var url = "list.html?" + xlist + ":" + (year+"-"+(y+1))
-            var pfx = ''
-            var sfx = ''
-            if ((year == firstYear && y < firstMonth0) || (year == lastYear && y > lastMonth0)) {
-                pfx = '<i>'
-                sfx = '</i>'
+            var yyyymm = (year+"-"+(y+1))
+            var url = "list.html?" + xlist + ":" + yyyymm
+            var yyyy0m = y < 9 ? (year+"-0"+(y+1)) : yyyymm
+            if (monthly_emails[yyyy0m]) {
+                var count = monthly_emails[yyyy0m]
+                cale += "<a href='" + url + "' onclick='return false;'><label id='calmonth_" + yyyymm + "' style='width: 80px; float: left;cursor: pointer;' class='label label-default label-hover' onclick='toggleEmail(" + year + ", " + (y + 1) + ");' >" + months[y] + ' ('+ count + ')' + "</label></a><br/>"
             }
-            cale += "<a href='" + url + "' onclick='return false;'><label id='calmonth_" + (year+"-"+(y+1)) + "' style='width: 80px; float: left;cursor: pointer;' class='label label-default label-hover' onclick='toggleEmail(" + year + ", " + (y + 1) + ");' >" + pfx + months[y] + sfx + "</label></a><br/>"
         }
-        cale += "</div>"
-        dp.innerHTML += cale
+        if (cale != '') {
+            cale += "</div>"
+            dp.innerHTML += "<label onmouseout='this.setAttribute(\"class\", \"label label-success\");'  onmouseover='this.setAttribute(\"class\", \"label label-warning\");' onclick='toggleCalendar(" + year + ");' class='label label-success' style='float: left; width: 110px; font-size: 11pt; cursor: pointer'>" + year + "</label><br/>"
+            var calehdr = "<div style='float: left; width: 80%; display: " + n + "; padding-left: 15px; margin-bottom: 15px;' id='cal_" + year + "'>"
+            dp.innerHTML += calehdr + cale
+        }
     }
     
     // Build the mobile version (dropdown)
diff --git a/test/generatortest.py b/test/generatortest.py
index 16963a2..0078b5c 100755
--- a/test/generatortest.py
+++ b/test/generatortest.py
@@ -45,7 +45,6 @@
 GENS=generators.generator_names()
 
 archie = archiver.Archiver(parse_html = parseHTML)
-fake_args = namedtuple('fakeargs', ['verbose', 'ibody'])(False, None)
 
 
 for arg in sys.argv[1:]:
@@ -68,9 +67,9 @@
                             break
                         if 'gen' in script:
                             print("Generator %s" % script['gen'])
-                            archiver.archiver_generator = script['gen']
+                            archie.generator = script['gen']
                         message = next(messages)
-                        json, contents, _msgdata, _irt = archie.compute_updates(fake_args, list_override, private, message)
+                        json, contents, _msgdata, _irt = archie.compute_updates(list_override, private, message)
                         error = 0
                         for key in script:
                             if key == 'gen':
@@ -89,8 +88,8 @@
         for message in messages:
             print(message.get_from())
             for gen in GENS:
-                archiver.archiver_generator = gen
-                json, contents, _msgdata, _irt = archie.compute_updates(fake_args, list_override, private, message)
+                archie.generator = gen
+                json, contents, _msgdata, _irt = archie.compute_updates(list_override, private, message)
                 print("%15s: %s" % (gen,json['mid']))
     elif arg.endswith('.eml'): # a single email
         for gen in GENS:
diff --git a/test/generatortest.yaml b/test/generatortest.yaml
index 863da00..8664497 100644
--- a/test/generatortest.yaml
+++ b/test/generatortest.yaml
@@ -50,16 +50,15 @@
      message-id: <e5e1d91dd42244d08bcec733b45ce538@git.apache.org>
    - mid: 7a8cd379babe8e83d58e429cdeadd3beeb060e47322f8e965d3acecc@1464785912@<commits.ponymail.apache.org>
      message-id: <7bb823a6fbf645e0943759e29ca2c67e@git.apache.org>
-   - mid: 85640e4075764900da2e734cd2a123f7909f0baeebafd0e4fdb3c1c5@<commits.ponymail.apache.org>
+   - mid: 788134ffd7d2b6630fefac41221e060c58fab35f3ffeb05978291ec0@<commits.ponymail.apache.org>
      message-id: <pony-f85cc4d67b0b00a9017135e4201b9225b5053fac-bb226b0ffe10c100b2a009d71a3fda7245b9865a@commits.ponymail.apache.org>
-     gen: medium_original
-   - mid: 4cba4ac4c1d66aeff438f0dea8ed0db1cd3a25680342ca6d12bb188b@<commits.ponymail.apache.org>
+     gen: medium # was medium_original, but that has been dropped
+   - mid: 660c3348c3ac301021c8d69c1a7c71549771c169d3d7efe9a94ef853@<commits.ponymail.apache.org>
      message-id: <951139456f694d2a831e0de670da47b0@git.apache.org>
-   - mid: 99e1a40d3ce06a1e09cc8a27e898deedee62455b8c7ad7359dd4323c@<commits.ponymail.apache.org>
+   - mid: 3ccf3f24f01ea81ff766567ea781b1ac78c2c7136a727437003a4a08@<commits.ponymail.apache.org>
      message-id: <bee810c90e4f41eaa2ce6fdad0449d39@git.apache.org>
    - mid: 51dfe8a64b8cb5c016db95e4bb164faf51e6b2d556b4c255a125f7ee@<commits.ponymail.apache.org>
      message-id: <af0cda1e818645039e2a3ea3cc655925@git.apache.org>
-     gen: medium
    - mid: bf458d3873aedae55431294f60326d85456c3a162b552db5fb0c8641@<commits.ponymail.apache.org>
      message-id: <662ce76c2bc54822a3617bcd8154220b@git.apache.org>
    - mid: b39348316bae9da1939c490b2feb911f8c600bb53920f61de76e593e@<commits.ponymail.apache.org>
diff --git a/test/requirements.txt b/test/requirements.txt
new file mode 100644
index 0000000..49fa8c3
--- /dev/null
+++ b/test/requirements.txt
@@ -0,0 +1,9 @@
+# for pytest/unit tests
+PyYAML~=5.3.1
+elasticsearch-dsl>=5.0.0
+elasticsearch~=5.0.0
+certifi~=2020.6.20
+chardet~=3.0.4
+netaddr~=0.8.0
+formatflowed~=2.0.0
+html2text~=2019.8.11
diff --git a/test/stats_cli.lua b/test/stats_cli.lua
new file mode 100755
index 0000000..b4c4069
--- /dev/null
+++ b/test/stats_cli.lua
@@ -0,0 +1,126 @@
+#!/usr/bin/env lua
+
+--[[
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+]]--
+
+-- Allow CLI testing of stats.lua query parameters
+-- Invoke with pairs of parameters, being the key and value, e.g.
+-- lua test_stats.lua list dev domain ponymail.apache.org q 'a/b' d lte=1y emailsOnly 1
+-- Note that parameters with no values are given the default value of "1" by mod_lua
+-- set MODE=inspect to print output query from inspect, otherwise print as JSON suitable for further processing
+
+-- Update path so we can always find the api/*.lua scripts
+local self=arg[0] or './dummy' -- get path to self
+local pfx=self:match("^.*/") or "" -- extract the path
+-- finally update package path (also adding current dir)
+package.path = package.path .. ";" .. pfx .. "../site/api/?.lua"   -- path to api dir
+package.path = package.path .. ";" .. pfx .. "/?.lua" -- current dir
+
+local inspect = require 'inspect'
+local http = require 'socket.http'
+local mock = require 'mock_r'
+local JSON = require 'cjson'
+require 'stats' -- local makes no difference here
+
+local _CACHE = {} -- capture output
+
+-- override http request so can capture the query
+http.request = function(url, data)
+  -- capture HTTP parameters (assume only called once)
+  _CACHE.url = url
+  _CACHE.querydata = JSON.decode(data)
+  -- return simplest result that satisfies stats.lua
+  result = [[
+{
+  "hits" : {
+  "total" : 0,
+  "hits" : [ ]
+  }
+}
+]]
+  return result, 200
+end
+
+
+local r = mock.r
+
+-- disable years active check
+r.ivm_get = function(r, key)
+  return JSON.encode({ pubfirst = 0, publast = 0})
+end
+
+-- collect output (assume only one call to puts)
+r.puts = function(r, ...) _CACHE.reply = JSON.decode(...) end
+
+-- TODO
+r.escape_html = function(r, val)
+  -- < > & are definitely escaped by the real escape_html
+  return val:gsub('>', '&gt;'):gsub('<', '&lt;'):gsub('&', '&amp;')
+end
+
+-- override the parse-args function so it returns our test data
+r.parseargs = function(r)
+  return _CACHE.args
+end
+
+
+local function test(args)
+  local output = {
+    quick = true, -- disable most queries
+  }
+  -- merge in user data
+  for k,v in pairs(args) do output[k] = v end  
+  _CACHE.args = output -- save the args
+  _CACHE.status = handle(r)
+  return _CACHE
+end
+
+local argc = #arg
+if argc == 0 -- assume reading lines of JSON strings
+then
+  for line in io.lines()
+  do
+    jzon = JSON.decode(line)
+    jzon.quick = true -- disable most queries
+    _CACHE.args = jzon
+    _CACHE.status = handle(r)
+    print(JSON.encode(_CACHE)) 
+    io.flush()     
+  end
+elseif argc % 2 == 0
+then
+  local data = {}
+  for i = 1,argc,2 do
+    data[arg[i]] = arg[i+1]
+  end
+  res = test(data)
+  if os.getenv("MODE") == "inspect" then
+    print(inspect(res["querydata"]))
+  else
+    print(JSON.encode(res))
+  end
+elseif argc == 1 -- assume JSON string
+then
+  jzon = JSON.decode(arg[1])
+  jzon.quick = true -- disable most queries
+  _CACHE.args = jzon
+  _CACHE.status = handle(r)
+  print(JSON.encode(_CACHE))
+else
+  print("Need even arg count")
+  os.exit(1)
+end
diff --git a/tools/import-mbox.py b/tools/import-mbox.py
index 28c0363..59f94f8 100755
--- a/tools/import-mbox.py
+++ b/tools/import-mbox.py
@@ -161,7 +161,7 @@
                             bmd = bf.read()
                             bf.close() # explicit early close
                             bmd = gzip.decompress(bmd)
-                            tmpfile = tempfile.NamedTemporaryFile(mode='w+b', buffering=1, delete=False)
+                            tmpfile = tempfile.NamedTemporaryFile(mode='w+b', delete=False)
                             tmpfile.write(bmd)
                             tmpfile.flush()
                             tmpfile.close()
@@ -514,7 +514,7 @@
         for mlist in re.finditer(ns, data):
             ml = mlist.group(1)
             mldata = urlopen("%s%s" % (source, ml)).read()
-            tmpfile = tempfile.NamedTemporaryFile(mode='w+b', buffering=1, delete=False)
+            tmpfile = tempfile.NamedTemporaryFile(mode='w+b', delete=False)
             try:
                 if ml.find(".gz") != -1:
                     mldata = gzip.decompress(mldata)