Merge pull request #530 from lewismc/ISSUE-529
ISSUE-529 Update README with new ponymail-foal project reference
diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml
index f8549ad..d78424e 100644
--- a/.github/workflows/pythonpackage.yml
+++ b/.github/workflows/pythonpackage.yml
@@ -14,18 +14,28 @@
steps:
- uses: actions/checkout@master
- - uses: actions/checkout@master
with:
+ persist-credentials: false
+ - uses: actions/checkout@master
+ with:
+ persist-credentials: false
repository: apache/incubator-ponymail-unit-tests
path: pmtests
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v1
with:
- version: ${{ matrix.python-version }}
+ python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
- pip install -r requirements.txt
+ pip install -r test/requirements.txt
+ - name: Set up LUA
+ # This is the commit for v8.0.0 (current at time of approval by INFRA)
+ uses: leafo/gh-actions-lua@ea0ae38722c0b45aa4e770f7c4a650c6b26800d0
+ with:
+ luaVersion: "5.2"
+ - name: Basic Test with LUA
+ run: lua -v
# - name: Test with pytest
# run: |
# pip install pytest
@@ -35,3 +45,7 @@
sed -e 's/# cropout:/cropout:/' tools/ponymail.cfg.sample >tools/ponymail.cfg
cd pmtests
python runall.py --root ..
+ - name: Generator tests
+ run: |
+ cd test
+ python generatortest.py generatortest.yaml
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 218fe8e..b6cf115 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,12 @@
## Changes in 0.12:
+- Bug: stats.lua dfrom and dto parameters are not documented in API (#544)
+- Bug: atom.lua is not documented (#538)
+- Bug: notifications.lua 'seen' option is not documented (#539)
+- Bug: &since option for stats.lua is not documented (#537)
+- Bug: PM setup does not support ES 6 - undocumented (#536)
+- Bug: buffering=1 not allowed with binary files (#534)
+- Bug: Invalid mailing list address supplied (#516)
+- Enh: stats.lua could return monthly stats to show in calendar (#532)
- Bug: archiver.py msgbody() function issues (#244 and #463)
- Bug: archiver.py: convertToWrapped expects bytes (#462)
- Bug: convertToWrapped expects a bytestring (#518)
diff --git a/README.md b/README.md
index c5d4316..a82f0ff 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,5 @@
+_NB: Development focus has shifted towards our [Pony Mail codename Foal](https://github.com/apache/incubator-ponymail-foal) replacement suite.<br/>Suggestions for new features should be focused on that repository instead._
+
# Apache Pony Mail (Incubating)
**Development on this project has shifted to [incubator-ponymail-foal](https://github.com/apache/incubator-ponymail-foal)**. This repository is now deprecated.
@@ -15,7 +17,8 @@
-See [https://lists.apache.org](https://lists.apache.org) for a demo.
+See [https://lists.apache.org](https://lists.apache.org) for a demo (this now runs the
+[Foal](https://github.com/apache/incubator-ponymail-foal) suite).
Pony Mail works in both public, private and mixed-mode, allowing you
to have one unified place for all your communication, both public and
@@ -44,7 +47,7 @@
### Requirements: ###
* Linux operating system (tested on Ubuntu, Debian, Fedora and CentOS - Windows or OS/X may work)
-* ElasticSearch backend (2.1 minimum)
+* ElasticSearch backend (2.1 minimum, 6.x maximum. Does not support 7.x+)
* Apache HTTP Server frontend with mod_lua loaded OR
* Nginx with nginx-extras (ng-lua module) AND lua-apr installed
* Python 3.x for importing (with elasticsearch and formatflowed via pip)
diff --git a/docs/API.md b/docs/API.md
deleted file mode 100644
index e8eba8f..0000000
--- a/docs/API.md
+++ /dev/null
@@ -1,225 +0,0 @@
-# Pony Mail Archive API
-
-### Fetching a specific email:
-
-Usage:
-`GET /api/email.lua?id=$mid[&attachment=true&file=$hash]`
-
-Parameters: (cookie may be required)
- - $mid: The email ID or Message-ID: header
- - $hash: the file attachment hash
-
-Response example:
-
-~~~
-{
- "references": "<git-pr-18-any23@git.apache.org>",
- "from_raw": "lewismc <git@git.apache.org>",
- "message-id": "<20150905153416.0CDCFDFE66@git1-us-west.apache.org>",
- "@import_timestamp": "2015/10/04 09:52:41",
- "body": "Body of email here...",
- "request_id": "06b318af97ca96c115e878c14d0814a53407751c31388410421c1751@1441467256@<dev.any23.apache.org>",
- "@version": 1,
- "attachments": {},
- "list": "<dev.any23.apache.org>",
- "date": "2015/09/05 17:34:16",
- "from": "lewismc <git@git.apache.org>",
- "gravatar": "a676c0bf448fcd49f588249ead719b4c",
- "in-reply-to": "<git-pr-18-any23@git.apache.org>",
- "epoch": 1441467256,
- "subject": "[GitHub] any23 pull request: Gsoc 2015 Microformats2",
- "mid": "06b318af97ca96c115e878c14d0814a53407751c31388410421c1751@1441467256@<dev.any23.apache.org>",
- "private": false,
- "tid": "06b318af97ca96c115e878c14d0814a53407751c31388410421c1751@1441467256@<dev.any23.apache.org>",
- "list_raw": "<dev.any23.apache.org>"
-}
-
-Note: date and epoch are in UTC
-
-~~~
-
-
-### Fetching list data
-Usage:
-`GET /api/stats.lua?list=$list&domain=$domain[&d=$timespan][&q=$query][&header_from=$from][&header_to=$to][&header_subject=$subject][&header_body=$body][&quick][&emailsOnly][&s=$s&e=$e]`
-
-Parameters:
-
- - $list: The list prefix (e.g. `dev`). Wildcards may be used
- - $domain: The list domain (e.g. `httpd.apache.org`). Wildcards may be used
- - $timespan: A [timespan](#Timespans) value
- - $s: yyyy-mm start of month (day 1)
- - $e: yyyy-mm end of month (last day)
- - $query: A search query (may contain wildcards or negations):
- - `foo`: Find all documents containing `foo` in headers or body
- - `-foo`: Find all documents NOT containing `foo`.
- - `foo*`: Find all documents containing `foo`, `fooa`, `foob` etc
- - $from: Optional From: address
- - $to: Optional To: address
- - $subject: Optional Subject: line
- - $body: Optional body text
-
-Options:
-
- - quick: only return list of email epochs
- - emailsOnly: only return list of emails; omit thread structure, top 10 participants and word-cloud
-
-Response example:
-
-~~~
-{
- "took": 437179,
- "firstYear": 2015,
- "emails": {
- {
- "list_raw": "<dev.ponymail.apache.org>",
- "gravatar": "66cf545ca7a1b8f595282bb9d8a59657",
- "id": "b1d6446f5cc8f4846454cbabc48ddb08afbb601a77169f8e32e34102@<dev.ponymail.apache.org>",
- "epoch": 1474883100,
- "subject": "Re: Missing tag for 0.9 release",
- "message-id": "<7f249f5e-e422-68a5-d57f-bfce585e638e@apache.org>",
- "private": false,
- "irt": "<CAOGo0VYrCOR=820LSDZA=czc==SOwCaRKasaEvVuxtUEXp9SDQ@mail.gmail.com>",
- "from": "Daniel Gruno <h...@apache.org>",
- "attachments": 0
- },...
- },
- "no_threads": 10,
- "domain": "ponymail.info",
- "participants": {
- {
- "count": 3,
- "name": "Daniel Gruno",
- "gravatar": "66cf545ca7a1b8f595282bb9d8a59657",
- "email": "hu...@apache.org"
- }, ...
- },
- "lastYear": 2015,
- "name": "dev",
- "cloud": {...},
- "hits": 25,
- "thread_struct": {...},
- thread_struct":
- {
- "nest": 2,
- "children": {
- {
- "children": {
- {
- "children": {
- {
- "children": { },
- epoch: ...,
- tid: ...,
- nest: 1
- }
- },
- epoch: ...,
- tid: ...,
- nest: 2
- }
- },
- "epoch": 1474883100,
- "tid": "b1d6446f5cc8f4846454cbabc48ddb08afbb601a77169f8e32e34102@<dev.ponymail.apache.org>",
- "nest": 2
- }
- },
- epoch: ...,
- tid: ...,
- body: ...
- },
- "max": 5000,
- "searchlist": "<dev.ponymail.info>",
- "list": "dev@ponymail.info",
- "numparts": 0,
- "using_wc": false
-}
-~~~
-
-### <a name="Timespans"></a>Timespans
-
-Timespans supported by the &d= parameter.
-
- - d=yyyy-mm => equivalent to &s=yyyy-mm&e=yyyy-mm
- - d=lte=n[wMyd] (less than n[wMyd] ago, inclusive)
- - d=gte=n[wMyd] (more than n[wMyd] ago, inclusive)
- - d=.*dfr=yyyy-mm-dd.* (start date for search, inclusive)
- - d=.*dto=yyyy-mm-dd.* (end date for search, inclusive)
- - [wMyd] = weeks, Months, years, days
- - lte and gte are mutually exclusive
- - dfr and dto are normally both present
-
-### Fetching preferences and quick list overview
-Usage:
-`GET /api/preferences.lua[?logout][?associate=$email][?verify&hash=$hash][?removealt=$email][?save][?addfav=$list][?remfav=$list]`
-
-Parameters: (cookie required)
- - logout: Whether to log out of the system (optional)
- - associate=$email - associate the account with the $email address
- - verify&hash=$hash - verify an association request $hash
- - removealt=$email - remove an alternate $email address
- - save - save preferences
- - addfav=$list - add a favourite $list
- - remfav=$list - remove a favourite $list
-
-
-Response example:
-
-~~~
-{
- "lists": {
- "ponymail.info": {
- "user": 5,
- "dev": 36,
- "commits": 279
- }
- },
- "descriptions": {
- },
- "preferences": {
- "displayMode": "threaded",
- "hideStats": "no",
- "theme": "default",
- "notifications": "direct",
- "sortOrder": "forward",
- "compactQuotes": "yes",
- "fullname": "Daniel Gruno",
- "groupBy": "thread"
- },
- "took": 38487,
- "login": {
- "notifications": 0,
- "credentials": {
- "fullname": "Daniel Gruno",
- "email": "foo@bar.tld"
- }
- }
-}
-~~~
-
-
-### Fetching notifications for a logged in user
-Usage:
-`GET /api/notifications.lua`
-
-Parameters: `None` (cookie required)
-
-
-Response example:
-
-~~~
-{
- "notifications": {...}
-}
-~~~
-
-### Fetching a month's data as an mbox file
-Usage:
-`GET /api/mbox.lua?list=issues@ponymail.apache.org&date=2016-06`
-
-Response example:
-
-~~~
-TBA
-~~~
-
diff --git a/docs/ARCHIVING.md b/docs/ARCHIVING.md
deleted file mode 100644
index 75246ba..0000000
--- a/docs/ARCHIVING.md
+++ /dev/null
@@ -1,95 +0,0 @@
-# Archiving New Emails to Pony Mail #
-This document exists to extend the [general install guide](INSTALLING.md) provide examples on how to archive emails.
-
-__Note:__ If you plan on [importing old emails from an archive](IMPORTING.md),
-please set up the archiver __first__ so as to create an overlap of new emails
-coming in and old emails being imported. The system is designed to handle this
-without creating duplicate entries in the archive.
-
-## Mailman 2.x example:
-Set up a Pony Mail mail account/alias on a machine. This can be your local mail
-server, it can be the machine that Pony Mail is on (install sendmail or postfix
-etc there), or it can be any other machine with access to the ElasticSearch
-database that Pony Mail uses.
-
-
-### Pre-requisites
-If this is not the machine Pony Mail was installed on, you'll need to copy the
-tools/ directory from your Pony Mail installation to this machine and adjust
-ponymail.cfg to point to the right place for the database. You will also need
-Python 3 and the helper libraries installed
-(`pip3 install elasticsearch formatflowed netaddr`)
-
-### Create an alias:
-Set up a mail alias for public and private lists in `/etc/aliases` or similar method,
-and point them at the archiver script in tools/:
-
-~~~
-# You may need to add "--altheader delivered-to" to these commands, it varies
-foo-public: "|/usr/bin/python3 /path/to/tools/archiver.py"
-foo-private: "|/usr/bin/python3 /path/to/tools/archiver.py --private"
-~~~
-
-Once done, run `newaliases` to update your alias DB.
-
-### Subscribe the aliases to your mailing lists
-Use the mailman UI or CLI to subscribe foo-public@ to your public lists and
-foo-private to your private lists. Don't worry, the contents of private lists
-are hidden by default till the correct AAA scripting is set up.
-
-
-## ezmlm example:
-First, see the general introduction in the MM2 example, as this applies here as well.
-
-### Create an alias:
-Set up a dot-forward file for a public and a private alias:
-
-~~~
-.qmail-archive-public:
- "|/usr/bin/python3 /path/to/tools/archiver.py"
-
-.qmail-archive-private:
- "|/usr/bin/python3 /path/to/tools/archiver.py --private"
-~~~
-
-
-### Subscribe the aliases to your mailing lists
-Use the ezmlm CLI to subscribe your new aliases to the lists:
-`ezmlm-sub foolist/ archive-public@yourhost.tld`
-`ezmlm-sub secretlist/ archive-private@yourhost.tld`
-
-
-## Setting up AAA
-If you have an custom OAuth2 provider and a binary approach to private access
-(either/or), you can enable private access to people by having a key/value pair
-called `isMember` set to `true` in your JSON response from the OAuth server,
-provided it is set as an authority in config.lua. This will provide anyone
-defined as a member via OAuth full access to all private lists.
-
-If you use LDAP, you can modify the LDAP queries in the example AAA file to suit
-your organization.
-
-
-## Importing/Archiving HTML-only emails
-Should you need to import HTML-only emails into the archive, you may enable this
-with the `--html2text` command line arg. This requires that the `html2text` Python3 package
-is installed beforehand.
-
-## Munging list IDs
-If you need to rewrite list IDs on the fly as emails come in, you can use the debug.cropout
-setting for this (in `ponymail.cfg`).
-
-You can either use it to just crop away something:
-~~~
- [debug]
- # Remove 'foo' from all list IDs
- cropout: foo
-~~~
-
- Or you can use it as a regex substitution:
-~~~
- [debug]
- #Replace '*.bar.tld' with '*.blorg.bar.tld'
- cropout: <([a-z]+)\.bar\.tld> <\1.blorg.bar.tld>
-~~~
-
diff --git a/docs/BUILDING.md b/docs/BUILDING.md
deleted file mode 100644
index 9ea97b7..0000000
--- a/docs/BUILDING.md
+++ /dev/null
@@ -1,19 +0,0 @@
-# Building Pony Mail for Production #
-Most of Pony Mail is ready-for-deployment files that just need to be checked out
-in order to work. Some areas, such as the JavaScript needs to be combined by a script,
-as they have been split into several smaller files to make it easier to find and
-work on various elements of the rendering process.
-
-### Building the JavaScript chunks ###
-All JavaScript edits should be done to the `site/js/dev/*.js` files.
-Once done, you should run combine.sh in the `site/js/dev` directory
-to generate ponymail.js from the scripts in the dev dir:
-
- $cd site/js/dev
- $bash combine.sh
- Combining JS...
- Done!
- $
-
-You may choose to commit the initial JS changes first before
-committing the new combined JS, but that's up to you.
diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md
deleted file mode 100644
index 9d25f45..0000000
--- a/docs/CONTRIBUTING.md
+++ /dev/null
@@ -1,33 +0,0 @@
-
-# Contributing to Pony Mail #
-We'd LOVE it if more people would contribute to Pony Mail!
-Any form of contribution is most welcome, whether it be programming,
-documentation, evangelism, marketing, or helping out other users.
-
-Pony Mail is first and foremost a user interface, and as such, we
-are always on the lookout for user experiences. If you have used
-Pony Mail and have feedback or ideas you wish to share, please let
-us know either through an issue/PR here or on the mailing list.
-
-## Code ##
-
-The Pony Mail code is on GitHub at https://github.com/apache/incubator-ponymail
-and pull requests are welcome.
-
-## Mailing lists ##
-
-Developers list:
- - list: dev@ponymail.incubator.apache.org
- - subscribe addr: dev-subscribe@ponymail.incubator.apache.org
- - Online version: http://lists.apache.org/list.html?dev@ponymail.incubator.apache.org
-
-Issues list:
- - list: issues@ponymail.incubator.apache.org
- - subscribe addr: issues-subscribe@ponymail.incubator.apache.org
- - Online version: http://lists.apache.org/list.html?bugs@ponymail.incubator.apache.org
-
-## Chat ##
-
-IRC:
- - #ponymail on Freenode
-
diff --git a/docs/DESIGN-NOTES.md b/docs/DESIGN-NOTES.md
deleted file mode 100644
index 1feee34..0000000
--- a/docs/DESIGN-NOTES.md
+++ /dev/null
@@ -1,76 +0,0 @@
-# Design Notes
-
-This file is an attempt to summarise some of the design issues.
-
-## Database
-The project uses the ElasticSearch (ES) database to store the mails as individual documents.
-The database stores each mail to each list as a separate document.
-If the same mail was sent to multiple lists, then it exists as multiple documents in the database.
-
-ES requires that each distinct document has a unique id (MID).
-The MID is used to insert the document in the database, and can be used to fetch it.
-
-### Database design
-The mails are stored in two separate ES indexes:
-* "mbox" - this stores information about the document, plus the parsed content, and is used for searching and summary displays.
-* "mbox_source" - this is used to store the raw content of the document.
-The two versions of the document are linked by using the same MID.
-
-### Requirements for the MID
-As mentioned above, each different document must have a unique id (MID).
-This document may arrive as a single mail message, or be loaded from a collection such as an mbox file.
-
-Duplicate database entries can be avoided by ensuring that the same MID is calculated regardless of the input source.
-[If the same message is processed more than once, it then does not matter as only the last instance will be stored.]
-The MID format does not have to be transparent; it can be an opaque hash.
-
-### Generation of the MID
-The same message may be sent to multiple lists, so the message data alone is not sufficient to identify it uniquely.
-The same message may potentially be sent more than once to the same list,
-so the combination of message and listname is also not sufficient to identify a message.
-
-Many messages will have a Message-Id header which is intended to be unique to the message.
-However this may not be the case, and some messages do not have one.
-
-Many mailing list servers will allocate a squence number or other such id to each message they send.
-This should be unique for the list, assuming that sequence is not reset.
-
-Where the Message-Id and List Server Id both exist, they can be combined to generate a MID.
-[If the List Server Id is known to be unique, then that can potentially be used alone.]
-
-Where one or other id does not exist, then alternative means need to be used to generate the MID.
-The data used to do so must be present it all supported message sources.
-
-Algorithms for the generator remain TBA
-
-### Permalink requirements
-The application provides Permalinks which can later be used to refer to any document in the database.
-Once published, it is important that such links must continue to work.
-
-Links should be portable; i.e. if the raw messages are loaded into a new archive it should be possible
-to support existing published Permalinks.
-
-Multiple links may refer to the same document, however each link should refer to a single document.
-Ideally the Permalink should be relatively short; however that may conflict with the uniqueness requirement.
-
-It may be useful for the Permalink format to be relatively transparent.
-For example, a current ASF mod_mbox link looks like:
-
-http://mail-archives.apache.org/mod_mbox/ponymail-commits/201605.mbox/<1f73b4e0fc1a4fbbbfe4d155293c2f1a@git.apache.org>
-
-This includes a reference to the:
-- mailing list name (ponymail-commits)
-- month when mail was sent (201605.mbox)
-- the Message-Id (<1f73b4e0fc1a4fbbbfe4d155293c2f1a@git.apache.org>)
-
-This information should be sufficient to find the message in just about any mail-archive.
-
-Whereas vendor-specific links may be much shorter, but are only valid for the particular service.
-For example the equivalent Markmail link is:
-http://markmail.org/message/oanktcpxlxkmyora
-
-There may be use cases for both styles of link.
-
-### Permalink design
-TBA
-
diff --git a/docs/IMPORTING.md b/docs/IMPORTING.md
deleted file mode 100644
index 465cd28..0000000
--- a/docs/IMPORTING.md
+++ /dev/null
@@ -1,80 +0,0 @@
-# Importing Data to Pony Mail #
-Pony Mail supports many ways of importing your old mail archives via the
-`import-mbox.py` script. For command line argument tips, run `python3
-import-mbox.py --help`.
-
-Imports are digested equally every time (*), so you can
-import from the same source multiple times without creating duplicate emails in
-the archive. Both the archiver and the importer use the same digest method, so
-they can overlap. Usually, you'll want to set up the archiver first, and when
-emails start flowing through it, you'll use the importer to import older emails.
-
-## Importing from mod_mbox
-
-### Importing a single domain
-Provided you have the main mod_mbox page at https://your.tld/mod_mbox/ and your (sub)domain resources at
-https://your.tld/mod_mbox/$list-yourdomain/, you can import all lists from that domain using:
-
-`python3 import-mbox.py --source https://your.tld/mod_mbox/ --mod-mbox --project yourdomain`
-
-For a quick update, which only imports the last 2 months of mail, append the `--quick` flag.
-
-You can also import just a single list by specifying that list ID:
-
-`python3 import-mbox.py --source https://your.tld/mod_mbox/ --mod-mbox --project listname-yourdomain`
-
-### Importing an entire archive (multiple domains)
-To import an entire site, use the same command as above, but omit the `--project` flag
-
-`python3 import-mbox.py --source https://your.tld/mod_mbox/ --mod-mbox`
-
-### Setting the domain or list id properly in case of variance
-If your old archive varies in terms of list IDs across time, you can force harmonization by using the `--lid` or `--domain` flags:
-
-`python3 import-mbox.py --source https://your.tld/mod_mbox/ --mod-mbox --project listid-yourdomain --lid "<listid.yourdomain.tld>"`
-
-This should only be done one list at a time.
-
-## Importing from Pipermail
-To import from pipermail, you will have to run the import one list at a time. As with mod_mbox imports, you must specify a source, but use `--pipermail` instead of `--mod-mbox`:
-
-`python3 import-mbox.py --source https://your.tld/pipermail/foolist/ --pipermail`
-
-### Pipermail and html-only emails
-While you can convert HTML-only emails to text using `--html2text`, Pipermail has some peculiarities
-where it adds a text/plain message to these emails, thus preventing html2text from working. You can
-circumvent this by using the `--ignorebody "foo"` arg to ignore all text/plain bodies containing `foo`.
-
-While the `project` flag is not needed here, you may wish to specify the list ID for the import.
-
-## Importing from locally stored mbox files
-To import from one or more local mbox files, specify a filesystem path as the source:
-
-`python3 import-mbox.py --source /tmp/mylists/`
-
-This will recursively import all files with the extension '.mbox'.
-
-You can change the extension as follows:
-
-`python3 import-mbox.py --source /tmp/mylists/ --ext .mbx`
-
-To match all files with any non-empty extension:
-
-`python3 import-mbox.py --source /tmp/mylists/ --ext '.*'`
-
-To match files regardless of extension:
-
-`python3 import-mbox.py --source /tmp/mylists/ --ext ''`
-
-Or you can import a single file:
-
-`python3 import-mbox.py --source 2016-11.mbox`
-
-(This is supported in versions after 0.9)
-
-## Test archives
-We have a few test archives for those that wish to test large imports.
-They can be found in gzip format at [http://ponymail.info/mboxes/](http://ponymail.info/mboxes/)
-
-(*) The digest depends on the [archiver] generator setting in ponymail.cfg
-If that varies between imports, then duplicates will occur
diff --git a/docs/INSTALL.centos.md b/docs/INSTALL.centos.md
deleted file mode 100644
index 99e2882..0000000
--- a/docs/INSTALL.centos.md
+++ /dev/null
@@ -1,141 +0,0 @@
-# Installing Pony Mail on CentOS 7.1: #
-This installation is a bit trickier, as CentOS does not have
-Python 3 or any of the lua modules in its default package system.
-
-Start by installing the following CentOS packages:
-
-- httpd
-- git
-- lua
-- lua-devel
-- gcc
-- gcc-c++
-- kernel-dev
-- unzip
-- openssl
-- openssl-devel
-- readline-devel
-
-~~~
-sudo yum install -y httpd git lua lua-devel gcc gcc-c++ kernel-devel unzip openssl openssl-devel readline-devel
-~~~
-
-
-Then, proceed to build LuaRocks (for lua deps):
-
-~~~
-wget http://luarocks.org/releases/luarocks-2.0.6.tar.gz
-tar zxvf luarocks-2.0.6.tar.gz
-cd luarocks-2.0.6
-./configure
-make
-sudo make install
-~~~
-
-Now build/install the required Lua modules:
-
-~~~
-sudo luarocks install lua-socket
-sudo luarocks install luasec OPENSSL_LIBDIR=/usr/lib64/
-sudo luarocks install lua-cjson
-~~~
-
-
-Configure, compile and install Python 3:
-
-~~~
-sudo yum groupinstall -y development
-sudo yum install -y zlib-dev sqlite-devel bzip2-devel xz-libs
-wget http://www.python.org/ftp/python/3.4.3/Python-3.4.3.tar.xz
-xz -d Python-3.4.3.tar.xz
-tar zvf Python-3.4.3.tar
-cd Python-3.4.3/
-./configure
-make
-sudo make altinstall
-~~~
-
-
-Install the required Python 3 modules:
-~~~
-sudo pip3.4 install elasticsearch formatflowed chardet netaddr
-~~~
-
-
-Install ElasticSearch:
-
-~~~
-sudo yum install -y java-1.7.0-openjdk-headless
-sudo rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch
-
- (The following is taken from the ElasticSearch online guide:)
-
- Add the following in your /etc/yum.repos.d/ directory in a file with a .repo suffix,
- for example elasticsearch.repo:
-
- [elasticsearch-1.7]
- name=Elasticsearch repository for 1.7.x packages
- baseurl=http://packages.elastic.co/elasticsearch/1.7/centos
- gpgcheck=1
- gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch
- enabled=1
-
-sudo yum update
-sudo yum install elasticsearch
-~~~
-
-
-Configure and start up ElasticSearch:
-
-~~~
-sudo sudo /bin/systemctl daemon-reload
-sudo sudo /bin/systemctl enable elasticsearch.service
-sudo /etc/init.d/elasticsearch start
-~~~
-
-
-Check out a copy of Pony Mail:
-~~~
-cd /var/www
-git clone https://github.com/apache/incubator-ponymail.git
-~~~
-
-
-Set up Pony Mail:
-~~~
-cd /var/www/ponymail/tools
-python3.4 setup.py
-[... answer questions asked by the setup script ...]
-~~~
-
-
-Set up Apache httpd by adding, for example, the following virtual host configuration:
-This differs from the normal installation (because of CentOS specifics), so beware
-
-~~~
-<VirtualHost *:80>
- LuaPackageCPath /usr/local/lib/lua/5.1/?.so
- LuaPackagePath /usr/local/share/lua/5.1/?.lua
- ServerName mylists.foo.tld
- DocumentRoot /var/www/ponymail/site
- AddHandler lua-script .lua
- LuaScope thread
- LuaCodeCache stat
- AcceptPathInfo On
-</VirtualHost>
-~~~
-
-(re)start apache:
-
-~~~
-sudo apachectl restart
-~~~
-
-Once this is done, you should now have a *working copy* of Pony Mail!
-
-You may wish to tweak the settings in `site/js/config.js` and your
-elasticsearch settings once Pony mail is up and running.
-
-Refer to the [General installation documentation](INSTALLING.md) for
-detailed information about archiving messages, OAuth, mail settings and
-much more.
diff --git a/docs/INSTALL.debian.md b/docs/INSTALL.debian.md
deleted file mode 100644
index 33b058d..0000000
--- a/docs/INSTALL.debian.md
+++ /dev/null
@@ -1,78 +0,0 @@
-# Installing Pony Mail on Debian Jessie: #
-Start by installing the following Debian packages:
-
-- apache2
-- git
-- lua-sec
-- lua-cjson
-- lua-socket
-- python3
-- python3-pip
-
-~~~
-sudo apt-get install apache2 git lua-cjson lua-sec lua-socket python3 python3-pip
-~~~
-
-Install the required Python 3 modules:
-~~~
-sudo pip3 install elasticsearch formatflowed netaddr
-~~~
-
-Install ElasticSearch:
-
-~~~
-sudo apt-get install openjdk-7-jre-headless
-wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
-echo "deb http://packages.elastic.co/elasticsearch/1.7/debian stable main" | sudo tee -a /etc/apt/sources.list.d/elasticsearch-1.7.list
-sudo apt-get update && sudo apt-get install elasticsearch
-~~~
-
-
-Check out a copy of Pony Mail:
-~~~
-cd /var/www
-sudo git clone https://github.com/apache/incubator-ponymail.git
-~~~
-
-Start up ElasticSearch:
-
-~~~
-sudo service elasticsearch start
-~~~
-
-Set up Pony Mail:
-~~~
-cd /var/www/ponymail/tools
-sudo python3 setup.py
-[... answer questions asked by the setup script ...]
-~~~
-
-
-Set up Apache httpd by adding, for example, the following virtual host configuration:
-
-~~~
-<VirtualHost *:80>
- ServerName mylists.foo.tld
- DocumentRoot /var/www/ponymail/site
- AddHandler lua-script .lua
- LuaScope thread
- LuaCodeCache stat
- AcceptPathInfo On
-</VirtualHost>
-~~~
-
-Enable mod_lua and start apache:
-
-~~~
-sudo a2enmod lua
-sudo service apache start
-~~~
-
-Once this is done, you should now have a *working copy* of Pony Mail!
-
-You may wish to tweak the settings in `site/js/config.js` and your
-elasticsearch settings once Pony mail is up and running.
-
-Refer to the [General installation documentation](INSTALLING.md) for
-detailed information about archiving messages, OAuth, mail settings and
-much more.
diff --git a/docs/INSTALL.fedora.md b/docs/INSTALL.fedora.md
deleted file mode 100644
index f27e1ee..0000000
--- a/docs/INSTALL.fedora.md
+++ /dev/null
@@ -1,112 +0,0 @@
-# Installing Pony Mail on Fedora 22: #
-
-Start by installing the following Fedora packages:
-
-- httpd
-- git
-- lua
-- lua-sec
-- lua-socket
-- python 3
-- luarocks
-
-~~~
-sudo dnf install -y httpd git lua lua-sec lua-socket python3 luarocks
-~~~
-
-Install the missing cjson package via luarocks:
-
-~~~
-sudo luarocks-5.3 install lua-cjson
-~~~
-
-Install the required Python 3 modules:
-~~~
-sudo pip3.4 install elasticsearch formatflowed chardet netaddr
-~~~
-
-
-Install ElasticSearch:
-
-~~~
-sudo dnf install -y java-1.8.0-openjdk-headless
-sudo rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch
-
- (The following is taken from the ElasticSearch online guide:)
-
- Add the following in your /etc/yum.repos.d/ directory in a file with a .repo suffix,
- for example elasticsearch.repo:
-
- [elasticsearch-1.7]
- name=Elasticsearch repository for 1.7.x packages
- baseurl=http://packages.elastic.co/elasticsearch/1.7/centos
- gpgcheck=1
- gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch
- enabled=1
-
-
-sudo dnf install -y elasticsearch
-~~~
-
-
-Configure and start up ElasticSearch:
-
-~~~
-sudo /bin/systemctl daemon-reload
-sudo /bin/systemctl enable elasticsearch.service
-sudo /etc/init.d/elasticsearch start
-~~~
-
-
-Check out a copy of Pony Mail:
-~~~
-cd /var/www
-sudo git clone https://github.com/apache/incubator-ponymail.git
-~~~
-
-
-Set up Pony Mail:
-~~~
-cd /var/www/ponymail/tools
-sudo python3.4 setup.py
-[... answer questions asked by the setup script ...]
-~~~
-
-
-Set up Apache httpd by adding, for example, the following virtual host configuration:
-This differs from the normal installation (because of CentOS specifics), so beware
-
-~~~
-<VirtualHost *:80>
- LuaPackageCPath /usr/lib/lua/5.3/?.so
- LuaPackagePath /usr/share/lua/5.3/?.lua
- ServerName mylists.foo.tld
- DocumentRoot /var/www/ponymail/site
- AddHandler lua-script .lua
- LuaScope thread
- LuaCodeCache stat
- AcceptPathInfo On
-</VirtualHost>
-~~~
-
-(re)start apache:
-
-~~~
-sudo apachectl restart
-~~~
-
-IF you have SELinux running, you need to allow httpd (apache) to
-be able to connect to remotes, otherwise Pony Mail won't work:
-
-~~~
-sudo setsebool -P httpd_can_network_connect 1
-~~~
-
-Once this is done, you should now have a *working copy* of Pony Mail!
-
-You may wish to tweak the settings in `site/js/config.js` and your
-elasticsearch settings once Pony mail is up and running.
-
-Refer to the [General installation documentation](INSTALLING.md) for
-detailed information about archiving messages, OAuth, mail settings and
-much more.
diff --git a/docs/INSTALL.ubuntu.md b/docs/INSTALL.ubuntu.md
deleted file mode 100644
index 95c47a2..0000000
--- a/docs/INSTALL.ubuntu.md
+++ /dev/null
@@ -1,94 +0,0 @@
-# Installing Pony Mail on Ubuntu 14.04 or 16.04: #
-Start by installing the following Ubuntu packages:
-
-- apache2
-- git
-- liblua5.2-dev
-- lua-sec
-- lua-cjson
-- lua-socket
-- python3
-- python3-pip
-- subversion
-
-~~~
-sudo apt-get install apache2 git liblua5.2-dev lua-cjson lua-sec lua-socket python3 python3-pip subversion
-~~~
-
-Install the required Python 3 modules:
-~~~
-sudo pip3 install elasticsearch formatflowed netaddr
-~~~
-
-Install ElasticSearch:
-~~~
-wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
-echo "deb http://packages.elastic.co/elasticsearch/2.x/debian stable main" | sudo tee -a /etc/apt/sources.list.d/elasticsearch-2.x.list
-sudo apt-get update && sudo apt-get install elasticsearch default-jre-headless
-~~~
-
-Compile and install mod_lua if necessary (httpd < 2.4.17 on Ubuntu):
-~~~
-sudo apt-get install apache2-dev
-svn co https://svn.apache.org/repos/asf/httpd/httpd/branches/2.4.x/modules/lua/
-cd lua/
-sudo apxs -I/usr/include/lua5.2 -cia mod_lua.c lua_*.c -lm -llua5.2
-~~~
-
-Check out a copy of Pony Mail:
-~~~
-sudo git clone https://github.com/apache/incubator-ponymail.git /var/www/ponymail
-~~~
-
-Configure Elasticsearch to automatically start during bootup. For Ubuntu <= 14.10:
-~~~
-sudo update-rc.d elasticsearch defaults 95 10
-~~~
-
-For Ubuntu >= 15.04:
-~~~
-sudo /bin/systemctl daemon-reload
-sudo /bin/systemctl enable elasticsearch.service
-~~~
-
-Start up ElasticSearch:
-
-~~~
-sudo service elasticsearch start
-~~~
-
-Set up Pony Mail:
-~~~
-cd /var/www/ponymail/tools
-sudo python3 setup.py
-[... answer questions asked by the setup script ...]
-~~~
-
-
-Set up Apache httpd by adding, for example, the following virtual host configuration (e.g. in `/etc/apache2/sites-enabled/000-default.conf`):
-~~~
-<VirtualHost *:80>
- ServerName mylists.foo.tld
- DocumentRoot /var/www/ponymail/site
- AddHandler lua-script .lua
- LuaScope thread
- LuaCodeCache stat
- AcceptPathInfo On
-</VirtualHost>
-~~~
-
-Enable mod_lua and start apache, if not already enabled:
-
-~~~
-sudo a2enmod lua
-sudo service apache start
-~~~
-
-Once this is done, you should now have a *working copy* of Pony Mail!
-
-You may wish to tweak the settings in `site/js/config.js` and your
-elasticsearch settings once Pony mail is up and running.
-
-Refer to the [General installation documentation](INSTALLING.md) for
-detailed information about archiving messages, OAuth, mail settings and
-much more.
diff --git a/docs/INSTALLING.md b/docs/INSTALLING.md
deleted file mode 100644
index 3f3fb4f..0000000
--- a/docs/INSTALLING.md
+++ /dev/null
@@ -1,215 +0,0 @@
-# Installing Pony Mail #
-
-
-If your distro is on this list, please refer to that specific document
-for detailed package installation instructions:
-
-- [Debian (Jessie) Installation Instructions](INSTALL.debian.md)
-- [Ubuntu (14.04) Installation Instructions](INSTALL.ubuntu.md)
-- [CentOS (7.1) Installation Instructions](INSTALL.centos.md)
-- [Fedora (22) Installation Instructions](INSTALL.fedora.md)
-
-Otherwise, read the next two chapters:
-
-## Pre-requisites ##
-You will need the following software installed on your machine:
-
-- ElasticSearch >= 2.1
-- Python 3.x for the archiver plugin (setup.py will handle dependencies) and importer
-- Python `html2text` package (GPLv3) if you wish to archive HTML-only mails (remember to add the `--html2text` command line arg)
-- Apache HTTP Server 2.4.x with mod_lua (see http://modlua.org/gs/installing if you need to build mod_lua manually)
-- Lua >=5.1 with the following modules: cjson, luasec, luasocket
- (Note: Lua 5.3 is not currently supported by httpd mod_lua or luasocket)
-
-
-## Download and Install ##
-
-- Download the git repo: `git clone https://github.com/apache/incubator-ponymail.git`
-- Start ElasticSearch on the machine it needs to run on.
-- Run setup.py in the `tools` dir:
-```
- $cd tools
- $python3 setup.py
- ...[follow instructions in the setup script]
-```
-- Edit `site/js/config.js` to suit your needs (usually very little editing is needed)
-
-
-
-### Using auth for ElasticSearch ###
-If your ElasticSearch instance requires authentication for the importer/archiver, please
-add the following lines in the `elasticsearch` block of `ponymail.cfg` once generated:
-
-~~~
-user: [username for ES]
-password: [password for ES]
-~~~
-
-
-### Using Apache HTTP Server: ###
-- Set up a VirtualHost block in Apache httpd that points to the `site/` directory in Pony Mail
-- Add the configuration snippets from `configs/ponymail_httpd.conf` to the virtual host
-- Start Apache httpd to enable the user-facing interface
-
-### Using nginx: ###
-- To use nginx, you will also need to install the `lua-apr` module from your distro.
-- Set up a Server block in nginx that points to the `site/` directory in Pony Mail
-- Add the configuration snippets from `configs/ponymail_nginx.conf` to the server config
-- Start nginx to enable the user-facing interface
-
-
-## Setting up the archiver ##
-First off, you will need both `tools/archiver.py` and the generated
-`tools/ponymail.cfg` present on the machine that your mail server runs on. This
-machine should also have access to the ElasticSearch backend.
-
-If your mailing list supports feeding emails to a program, feed the incoming new
-emails to `python3 /path/to/tools/archiver.py` and it will use STDIN as the
-transport mechanism. If you are simply using aliases or dot-forwards and no ML
-system, you can add (for example) `"|/usr/bin/python3
-/path/to/tools/archiver.py"` to your alias file to enable archiving. If you are
-not using a Mailing List manager, you will need to tell Pony Mail which email
-header determines the list ID using the --altheader argument, for instance:
-```
- foolist: "|/usr/bin/python3 /path/to/tools/archiver.py --altheader delivered-to"
- foolist-private: "|/usr/bin/python3 /path/to/tools/archiver.py --altheader delivered-to --private"
-```
-
-If you are using MailMan 3, you can add archiver.py as an archive by following the instructions inside the python script:
-- Copy the archiver.py file to `$mailman_plugin_dir/mailman_ponymail/__init__.py`
-- Copy ponymail.cfg to the same dir (for ES configuration)
-- Enable the module by adding the following to your `mailman.cfg` file::
-```
- [archiver.ponymail]
- # Pony Mail
- class: mailman_ponymail.Archiver
- enable: yes
-```
-
-For older mailing list systems such as Mailman 2 and ezmlm, you can also
-tak a look at our [archiving examples](ARCHIVING.md) page for pointers.
-
-## Public versus private lists ##
-In MailMan 3, this should be auto-detected and is not a concern.
-When using other ML systems via piping to STDIN, you should add
-the --private arg to the python script to mark an email as private:
-```
- foolist-private: "|/usr/bin/python3 /path/to/tools/archiver.py --private"
- foolist-public: "|/usr/bin/python3 /path/to/tools/archiver.py"
-```
-
-## Importing old data into Pony Mail
-See [this guide](IMPORTING.md) for details on how to import old archives into Pony Mail.
-
-## Bulk editing lists ##
-You can use `edit-list.py` to perform bulk operations:
-- Rename lists
-- Mark entire lists are private or public
-
-Run `python3 edit-list.py --help` for CLI args.
-
-
-## Setting up OAuth for Pony Mail ##
-If you want people to be able to log in and reply via the Web UI, you
-can specify an OAuth provider.
-
-
-### Setting up an OAuth provider ###
-Pony Mail comes with a few default OAuth examples in `site/js/config.js`, such
-as ASF Oauth and Google OAuth. You can enable these by uncommenting the lines in
-question, or set up your own OAuth portal to handle things. This is a standard
-OAuth that expects the backend to supply the following JSON data on success:
-
-~~~
- {
- "fullname": "The full name of the authed user",
- "email": "The user's email address",
- "uid": "(optional) The unique user ID of the logged in user (for instance, LDAP UID)",
- "isMember": true/false (optional, specifies whether the person is a privileged user with access to all lists)
- }
-~~~
-
-For private list browsing, Pony Mail supplies an example AAA library in
-`site/api/lib/aaa.lua` that does LDAP lookups to determine which groups a person
-belongs to, and thus which lists said person has access to. The AAA example is
-modelled on the Apache LDAP structure, so you may wish to change this to suit
-your need. We have [several simple AAA examples](../aaa_examples/) in the
-`aaa_examples` directory.
-
-If you are looking for an OAuth portal to provide users access to private lists
-in the archive, you will need to add the OAuth domain to config.admin_oauth in
-config.lua:
-
-~~~
- admin_oauth = { 'myoauth.foo.tld', '*.oauthprovider.com', 'etc' }
-~~~~
-
-If not specified in config.lua, OAuth will only provide users with a place to
-store settings and notifications, and - provided your mail server is set to accept
-this - a place to reply to emails in the archive.
-
-
-#### Using GitHub OAuth and other client-secret providers ####
-If your OAuth provider requires a client secret, you can specify this in `site/api/lib/config.lua`, as this GitHub example shows:
-
-~~~
- oauth_fields = {
- github = {
- client_secret = "abcdef1",
- client_id = "abcdef2",
- oauth_token = "https://github.com/login/oauth/access_token"
- }
- }
-~~~
-
-This essentially overrides `config.js` but without showing the data to anyone outside the server.
-
-### Whitelisting replies via the Web UI ###
-To have Pony Mail accept replies done via the Web UI, you must make sure
-that `site/api/lib/config.lua` contains the appropriate string (or array of strings) matching the domain(s) you wish to allow new email for. To allow replies to everything, set this to `* `(NOT RECOMMENDED).
-You can also allow based on GLOBs or an array of accepted domains and sub-domains:
-
-~~~
- accepted_domains = "*" -- This would allow posts to any email address, baaaad choice.
- accepted_domains = "foo.org" -- Allow only to *@foo.org
- accepted_domains = "*.foo.org" -- Allow only posts to *@*.foo.org, but not *@foo.org
- accepted_domains = { "foo.org", "*.foo.org" } -- Allow posts both to *.foo.org and foo.org
-~~~
-
-
-### Setting email footers ###
-It is possible to set email footers in each email sent via the Web UI.
-This is done by configuring the `email_footer` variable in `site/api/lib/config.lua`.
-You may use the following variables in the footer:
-
-~~~
- $list: The mailing list being sent to (foo@bar.tld)
- $hostname: The hostname of the server
- $port: The port of the server (80, 443 etc)
- $msgid: The message ID of the email (for permalinks etc)
-~~~
-
-An example footer could be:
-
-~~~
- --------
- Sent via Pony Mail for $list.
- To view this list online, visit: https://my.tld/list.html?$list
- To view this email (and subsequent replies), visit:
- https://my.tld/thread.html/$msgid
- --------
-~~~
-
-
-### A note on email headers ###
-By default, headers such as to/cc are not shown in the normal email view.
-To enable these headers, set `full_headers` to `true` in the `site/api/lib/config.lua` file.
-
-### Lastly, a note about Message-ID (MID) generators
-The default MID generator is called 'medium' and digests the message
-body, timestamp and list-ID to generate the MID. There is also a 'short'
-that only digests the body, and a 'full' that uses the entire message as
-a bytestring to generate an ID. Medium is recommended for most setups
-(especially clustered setups), while full can be used for single-machine
-setups.
-N.B. At present, all the generators have issues, see (#176 #177 #178)
diff --git a/docs/README b/docs/README
deleted file mode 100644
index b5eeff9..0000000
--- a/docs/README
+++ /dev/null
@@ -1,14 +0,0 @@
-The documentation appears in the following files:
-
-API reference: API.md
-Archiving email into Pony Mail: ARCHIVING.md
-Building Pony Mail for Production: BUILDING.md
-Contributing to Pony Mail: CONTRIBUTING.md
-Imorting existing data into Pony Mail: IMPORTING.md
-Installing:
- CentOS (and RHEL): INSTALL.centos.md
- Debian: INSTALL.debian.md
- Fedora: INSTALL.fedora.md
- Ubuntu: INSTALL.ubuntu.md
- Anywhere else: INSTALLING.md
-
diff --git a/requirements.txt b/requirements.txt
index 49fa8c3..efa8e24 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,9 +1,8 @@
-# for pytest/unit tests
-PyYAML~=5.3.1
-elasticsearch-dsl>=5.0.0
-elasticsearch~=5.0.0
-certifi~=2020.6.20
-chardet~=3.0.4
-netaddr~=0.8.0
-formatflowed~=2.0.0
-html2text~=2019.8.11
+# The following modules are required.
+# It is believed that they have licences as shown
+# N.B. modules with license that are compatible with AL 2.0 should be included here
+# As such, html2text (GPL) and chardet (GPL) cannot be included
+certifi # MPL 2.0
+elasticsearch < 6.0.0 # AL2.0
+formatflowed # Python Software Foundation
+netaddr # BSD License, MIT License
diff --git a/site/api/lib/config.lua.sample b/site/api/lib/config.lua.sample
index 7a84a68..0acb615 100644
--- a/site/api/lib/config.lua.sample
+++ b/site/api/lib/config.lua.sample
@@ -37,6 +37,7 @@
-- no_association = {}, -- domains that are not allowed for email association
-- listsDisplay = 'regex', -- if defined, hide list names that don't match the regex
-- debug = false, -- whether to return debug information
- antispam = true -- Whether or not to add anti-spam measures aimed at anonymous users.
+ antispam = true, -- Whether or not to add anti-spam measures aimed at anonymous users.
+ noShowQuery = false, -- disallow return query in JSON result: true|false (default false)
}
return config
diff --git a/site/api/lib/elastic.lua b/site/api/lib/elastic.lua
index 4eacf5b..192cac6 100644
--- a/site/api/lib/elastic.lua
+++ b/site/api/lib/elastic.lua
@@ -62,8 +62,6 @@
local result, hc = http.request(url, js)
checkReturn(hc, ok404)
local json = JSON.decode(result)
- -- TODO should we return the http status code?
- -- This might be necessary if codes such as 404 did not cause an error
return json, hc
end
diff --git a/site/api/stats.lua b/site/api/stats.lua
index 3e82bdb..a6af005 100644
--- a/site/api/stats.lua
+++ b/site/api/stats.lua
@@ -25,6 +25,19 @@
local cross = require 'lib/cross'
local utils = require 'lib/utils'
+-- accumulate the monthly stats
+local function monthly_emails(buckets, target)
+ for _,v in pairs(buckets) do
+ if v.doc_count > 0 then
+ if target[v.key_as_string] then
+ target[v.key_as_string] = target[v.key_as_string] + v.doc_count
+ else
+ target[v.key_as_string] = v.doc_count
+ end
+ end
+ end
+end
+
local BODY_MAXLEN = config.stats_maxBody or 200
-- words to exclude from word cloud:
local EXCLUDE = config.stats_wordExclude or ".|..|..."
@@ -429,6 +442,13 @@
max = {
field = "epoch"
}
+ },
+ monthly_emails = {
+ date_histogram = {
+ field = "date",
+ interval = "month",
+ format = "yyyy-MM"
+ }
}
}
}
@@ -439,10 +459,12 @@
datespan = {}
datespan.pubfirst = nil
datespan.publast = nil
+ datespan.monthly_emails = {}
-- find public min and max (buckets will be empty if there are no matching lists)
for _, list in pairs(doc.aggregations.lists.buckets) do
for _, private in pairs(list.private.buckets) do
if private.key_as_string == "false" then
+ monthly_emails(private.monthly_emails.buckets, datespan.monthly_emails)
if (datespan.publast == nil) or (private.last.value > datespan.publast) then datespan.publast = private.last.value end
if (datespan.pubfirst == nil) or (private.first.value < datespan.pubfirst) then datespan.pubfirst = private.first.value end
end
@@ -459,16 +481,16 @@
for _, list in pairs(doc.aggregations.lists.buckets) do
for _, private in pairs(list.private.buckets) do
if private.key_as_string == "true" then
+ datespan.private = datespan.private or {}
+ datespan.private[list.key] = datespan.private[list.key] or {}
+ datespan.private[list.key].monthly_emails = datespan.private[list.key].monthly_emails or {}
+ monthly_emails(private.monthly_emails.buckets, datespan.private[list.key].monthly_emails)
local prvlast = private.last.value
if prvlast > datespan.publast then
- datespan.private = datespan.private or {}
- datespan.private[list.key] = datespan.private[list.key] or {}
datespan.private[list.key].last = prvlast
end
local prvfirst = private.first.value
if prvfirst < datespan.pubfirst then
- datespan.private = datespan.private or {}
- datespan.private[list.key] = datespan.private[list.key] or {}
datespan.private[list.key].first = prvfirst
end
end
@@ -484,8 +506,16 @@
local last = datespan.publast
for lid, prvdates in pairs(datespan.private or {}) do
if aaa.canAccessList(r, lid, account) then
- if prvdates.first and prvdates.first < first then first = prvdates.first end
- if prvdates.last and prvdates.last > last then last = prvdates.last end
+ -- merge the stats from the private list
+ for k,v in pairs(prvdates.monthly_emails) do
+ if datespan.monthly_emails[k] then
+ datespan.monthly_emails[k] = datespan.monthly_emails[k] + v
+ else
+ datespan.monthly_emails[k] = v
+ end
+ end
+ if prvdates.first and prvdates.first < first then first = prvdates.first end
+ if prvdates.last and prvdates.last > last then last = prvdates.last end
end
end
@@ -738,6 +768,7 @@
listdata.lastYear = datespan.lastYear
listdata.firstMonth = datespan.firstMonth
listdata.lastMonth = datespan.lastMonth
+ listdata.monthly_emails = datespan.monthly_emails
listdata.list = listraw:gsub("^([^.]+)%.", "%1@"):gsub("[<>]+", "")
listdata.emails = emls
listdata.hits = #emls
@@ -750,6 +781,10 @@
end
listdata.numparts = allparts
listdata.unixtime = os.time()
+ if get.showQuery and not config.noShowQuery then
+ listdata.squery = squery
+ listdata.sdata = get
+ end
r:puts(JSON.encode(listdata))
diff --git a/site/api/thread.lua b/site/api/thread.lua
index 741ac28..1fc0f9b 100644
--- a/site/api/thread.lua
+++ b/site/api/thread.lua
@@ -84,9 +84,6 @@
-- Try searching by mid if not found, for backward compat
if not doc or not doc.mid then
local docs = elastic.find("message-id:\"" .. r:escape(eid) .. "\"", 1, "mbox")
- if #docs == 1 then
- doc = docs[1]
- end
if #docs == 0 and #eid == utils.SHORTENED_LINK_LEN then
docs = elastic.find("mid:" .. r:escape(eid) .. "*", 1, "mbox")
end
diff --git a/site/js/dev/ponymail_assign_vars.js b/site/js/dev/ponymail_assign_vars.js
index f9252d3..058c950 100644
--- a/site/js/dev/ponymail_assign_vars.js
+++ b/site/js/dev/ponymail_assign_vars.js
@@ -20,7 +20,7 @@
// Thus lightening the load on the backend (caching and such)
var _VERSION_ = "0.12-SNAPSHOT" // Current version (as far as we know)
-var months = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']
+var months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
var d_ppp = 15; // results per page
var c_page = 0; // current page position for list view
var open_emails = [] // cache index for loaded emails
diff --git a/site/js/dev/ponymail_pagebuilder.js b/site/js/dev/ponymail_pagebuilder.js
index 8619aeb..1d6d30b 100644
--- a/site/js/dev/ponymail_pagebuilder.js
+++ b/site/js/dev/ponymail_pagebuilder.js
@@ -37,6 +37,7 @@
var lastYear = json.lastYear
var firstMonth0 = json.firstMonth - 1 // 0-based
var lastMonth0 = json.lastMonth - 1 // 0-based
+ var monthly_emails = json.monthly_emails
// Build the main calendar (desktop version)
var dp = document.getElementById('datepicker')
@@ -54,21 +55,23 @@
if (fyear == firstYear) {
n = "block"
}
- dp.innerHTML += "<label onmouseout='this.setAttribute(\"class\", \"label label-success\");' onmouseover='this.setAttribute(\"class\", \"label label-warning\");' onclick='toggleCalendar(" + year + ");' class='label label-success' style='float: left; width: 110px; font-size: 11pt; cursor: pointer'>" + year + "</label><br/>"
- var cale = "<div style='float: left; width: 80%; display: " + n + "; padding-left: 15px; margin-bottom: 15px;' id='cal_" + year + "'>"
+ var cale = ''
var em = (new Date().getFullYear() == year) ? new Date().getMonth() : 11;
for (var y = em; y >= 0; y--) {
- var url = "list.html?" + xlist + ":" + (year+"-"+(y+1))
- var pfx = ''
- var sfx = ''
- if ((year == firstYear && y < firstMonth0) || (year == lastYear && y > lastMonth0)) {
- pfx = '<i>'
- sfx = '</i>'
+ var yyyymm = (year+"-"+(y+1))
+ var url = "list.html?" + xlist + ":" + yyyymm
+ var yyyy0m = y < 9 ? (year+"-0"+(y+1)) : yyyymm
+ if (monthly_emails[yyyy0m]) {
+ var count = monthly_emails[yyyy0m]
+ cale += "<a href='" + url + "' onclick='return false;'><label id='calmonth_" + yyyymm + "' style='width: 80px; float: left;cursor: pointer;' class='label label-default label-hover' onclick='toggleEmail(" + year + ", " + (y + 1) + ");' >" + months[y] + ' ('+ count + ')' + "</label></a><br/>"
}
- cale += "<a href='" + url + "' onclick='return false;'><label id='calmonth_" + (year+"-"+(y+1)) + "' style='width: 80px; float: left;cursor: pointer;' class='label label-default label-hover' onclick='toggleEmail(" + year + ", " + (y + 1) + ");' >" + pfx + months[y] + sfx + "</label></a><br/>"
}
- cale += "</div>"
- dp.innerHTML += cale
+ if (cale != '') {
+ cale += "</div>"
+ dp.innerHTML += "<label onmouseout='this.setAttribute(\"class\", \"label label-success\");' onmouseover='this.setAttribute(\"class\", \"label label-warning\");' onclick='toggleCalendar(" + year + ");' class='label label-success' style='float: left; width: 110px; font-size: 11pt; cursor: pointer'>" + year + "</label><br/>"
+ var calehdr = "<div style='float: left; width: 80%; display: " + n + "; padding-left: 15px; margin-bottom: 15px;' id='cal_" + year + "'>"
+ dp.innerHTML += calehdr + cale
+ }
}
// Build the mobile version (dropdown)
diff --git a/site/js/ponymail.js b/site/js/ponymail.js
index 3c7e9d9..d814bc2 100644
--- a/site/js/ponymail.js
+++ b/site/js/ponymail.js
@@ -28,7 +28,7 @@
// Thus lightening the load on the backend (caching and such)
var _VERSION_ = "0.12-SNAPSHOT" // Current version (as far as we know)
-var months = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']
+var months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
var d_ppp = 15; // results per page
var c_page = 0; // current page position for list view
var open_emails = [] // cache index for loaded emails
@@ -2323,7 +2323,7 @@
// ML address: only accept valid mailing list name, domain or both
// return true if the address is valid
function valid_address(val) {
- return val.match(/^[-@A-Za-z.0-9]+$/);
+ return val.match(/^[-_@A-Za-z.0-9]+$/);
}
// Check for slow URLs every 0.1 seconds
@@ -3688,6 +3688,7 @@
var lastYear = json.lastYear
var firstMonth0 = json.firstMonth - 1 // 0-based
var lastMonth0 = json.lastMonth - 1 // 0-based
+ var monthly_emails = json.monthly_emails
// Build the main calendar (desktop version)
var dp = document.getElementById('datepicker')
@@ -3705,21 +3706,23 @@
if (fyear == firstYear) {
n = "block"
}
- dp.innerHTML += "<label onmouseout='this.setAttribute(\"class\", \"label label-success\");' onmouseover='this.setAttribute(\"class\", \"label label-warning\");' onclick='toggleCalendar(" + year + ");' class='label label-success' style='float: left; width: 110px; font-size: 11pt; cursor: pointer'>" + year + "</label><br/>"
- var cale = "<div style='float: left; width: 80%; display: " + n + "; padding-left: 15px; margin-bottom: 15px;' id='cal_" + year + "'>"
+ var cale = ''
var em = (new Date().getFullYear() == year) ? new Date().getMonth() : 11;
for (var y = em; y >= 0; y--) {
- var url = "list.html?" + xlist + ":" + (year+"-"+(y+1))
- var pfx = ''
- var sfx = ''
- if ((year == firstYear && y < firstMonth0) || (year == lastYear && y > lastMonth0)) {
- pfx = '<i>'
- sfx = '</i>'
+ var yyyymm = (year+"-"+(y+1))
+ var url = "list.html?" + xlist + ":" + yyyymm
+ var yyyy0m = y < 9 ? (year+"-0"+(y+1)) : yyyymm
+ if (monthly_emails[yyyy0m]) {
+ var count = monthly_emails[yyyy0m]
+ cale += "<a href='" + url + "' onclick='return false;'><label id='calmonth_" + yyyymm + "' style='width: 80px; float: left;cursor: pointer;' class='label label-default label-hover' onclick='toggleEmail(" + year + ", " + (y + 1) + ");' >" + months[y] + ' ('+ count + ')' + "</label></a><br/>"
}
- cale += "<a href='" + url + "' onclick='return false;'><label id='calmonth_" + (year+"-"+(y+1)) + "' style='width: 80px; float: left;cursor: pointer;' class='label label-default label-hover' onclick='toggleEmail(" + year + ", " + (y + 1) + ");' >" + pfx + months[y] + sfx + "</label></a><br/>"
}
- cale += "</div>"
- dp.innerHTML += cale
+ if (cale != '') {
+ cale += "</div>"
+ dp.innerHTML += "<label onmouseout='this.setAttribute(\"class\", \"label label-success\");' onmouseover='this.setAttribute(\"class\", \"label label-warning\");' onclick='toggleCalendar(" + year + ");' class='label label-success' style='float: left; width: 110px; font-size: 11pt; cursor: pointer'>" + year + "</label><br/>"
+ var calehdr = "<div style='float: left; width: 80%; display: " + n + "; padding-left: 15px; margin-bottom: 15px;' id='cal_" + year + "'>"
+ dp.innerHTML += calehdr + cale
+ }
}
// Build the mobile version (dropdown)
diff --git a/test/generatortest.py b/test/generatortest.py
index 16963a2..0078b5c 100755
--- a/test/generatortest.py
+++ b/test/generatortest.py
@@ -45,7 +45,6 @@
GENS=generators.generator_names()
archie = archiver.Archiver(parse_html = parseHTML)
-fake_args = namedtuple('fakeargs', ['verbose', 'ibody'])(False, None)
for arg in sys.argv[1:]:
@@ -68,9 +67,9 @@
break
if 'gen' in script:
print("Generator %s" % script['gen'])
- archiver.archiver_generator = script['gen']
+ archie.generator = script['gen']
message = next(messages)
- json, contents, _msgdata, _irt = archie.compute_updates(fake_args, list_override, private, message)
+ json, contents, _msgdata, _irt = archie.compute_updates(list_override, private, message)
error = 0
for key in script:
if key == 'gen':
@@ -89,8 +88,8 @@
for message in messages:
print(message.get_from())
for gen in GENS:
- archiver.archiver_generator = gen
- json, contents, _msgdata, _irt = archie.compute_updates(fake_args, list_override, private, message)
+ archie.generator = gen
+ json, contents, _msgdata, _irt = archie.compute_updates(list_override, private, message)
print("%15s: %s" % (gen,json['mid']))
elif arg.endswith('.eml'): # a single email
for gen in GENS:
diff --git a/test/generatortest.yaml b/test/generatortest.yaml
index 863da00..8664497 100644
--- a/test/generatortest.yaml
+++ b/test/generatortest.yaml
@@ -50,16 +50,15 @@
message-id: <e5e1d91dd42244d08bcec733b45ce538@git.apache.org>
- mid: 7a8cd379babe8e83d58e429cdeadd3beeb060e47322f8e965d3acecc@1464785912@<commits.ponymail.apache.org>
message-id: <7bb823a6fbf645e0943759e29ca2c67e@git.apache.org>
- - mid: 85640e4075764900da2e734cd2a123f7909f0baeebafd0e4fdb3c1c5@<commits.ponymail.apache.org>
+ - mid: 788134ffd7d2b6630fefac41221e060c58fab35f3ffeb05978291ec0@<commits.ponymail.apache.org>
message-id: <pony-f85cc4d67b0b00a9017135e4201b9225b5053fac-bb226b0ffe10c100b2a009d71a3fda7245b9865a@commits.ponymail.apache.org>
- gen: medium_original
- - mid: 4cba4ac4c1d66aeff438f0dea8ed0db1cd3a25680342ca6d12bb188b@<commits.ponymail.apache.org>
+ gen: medium # was medium_original, but that has been dropped
+ - mid: 660c3348c3ac301021c8d69c1a7c71549771c169d3d7efe9a94ef853@<commits.ponymail.apache.org>
message-id: <951139456f694d2a831e0de670da47b0@git.apache.org>
- - mid: 99e1a40d3ce06a1e09cc8a27e898deedee62455b8c7ad7359dd4323c@<commits.ponymail.apache.org>
+ - mid: 3ccf3f24f01ea81ff766567ea781b1ac78c2c7136a727437003a4a08@<commits.ponymail.apache.org>
message-id: <bee810c90e4f41eaa2ce6fdad0449d39@git.apache.org>
- mid: 51dfe8a64b8cb5c016db95e4bb164faf51e6b2d556b4c255a125f7ee@<commits.ponymail.apache.org>
message-id: <af0cda1e818645039e2a3ea3cc655925@git.apache.org>
- gen: medium
- mid: bf458d3873aedae55431294f60326d85456c3a162b552db5fb0c8641@<commits.ponymail.apache.org>
message-id: <662ce76c2bc54822a3617bcd8154220b@git.apache.org>
- mid: b39348316bae9da1939c490b2feb911f8c600bb53920f61de76e593e@<commits.ponymail.apache.org>
diff --git a/test/requirements.txt b/test/requirements.txt
new file mode 100644
index 0000000..49fa8c3
--- /dev/null
+++ b/test/requirements.txt
@@ -0,0 +1,9 @@
+# for pytest/unit tests
+PyYAML~=5.3.1
+elasticsearch-dsl>=5.0.0
+elasticsearch~=5.0.0
+certifi~=2020.6.20
+chardet~=3.0.4
+netaddr~=0.8.0
+formatflowed~=2.0.0
+html2text~=2019.8.11
diff --git a/test/stats_cli.lua b/test/stats_cli.lua
new file mode 100755
index 0000000..b4c4069
--- /dev/null
+++ b/test/stats_cli.lua
@@ -0,0 +1,126 @@
+#!/usr/bin/env lua
+
+--[[
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+]]--
+
+-- Allow CLI testing of stats.lua query parameters
+-- Invoke with pairs of parameters, being the key and value, e.g.
+-- lua test_stats.lua list dev domain ponymail.apache.org q 'a/b' d lte=1y emailsOnly 1
+-- Note that parameters with no values are given the default value of "1" by mod_lua
+-- set MODE=inspect to print output query from inspect, otherwise print as JSON suitable for further processing
+
+-- Update path so we can always find the api/*.lua scripts
+local self=arg[0] or './dummy' -- get path to self
+local pfx=self:match("^.*/") or "" -- extract the path
+-- finally update package path (also adding current dir)
+package.path = package.path .. ";" .. pfx .. "../site/api/?.lua" -- path to api dir
+package.path = package.path .. ";" .. pfx .. "/?.lua" -- current dir
+
+local inspect = require 'inspect'
+local http = require 'socket.http'
+local mock = require 'mock_r'
+local JSON = require 'cjson'
+require 'stats' -- local makes no difference here
+
+local _CACHE = {} -- capture output
+
+-- override http request so can capture the query
+http.request = function(url, data)
+ -- capture HTTP parameters (assume only called once)
+ _CACHE.url = url
+ _CACHE.querydata = JSON.decode(data)
+ -- return simplest result that satisfies stats.lua
+ result = [[
+{
+ "hits" : {
+ "total" : 0,
+ "hits" : [ ]
+ }
+}
+]]
+ return result, 200
+end
+
+
+local r = mock.r
+
+-- disable years active check
+r.ivm_get = function(r, key)
+ return JSON.encode({ pubfirst = 0, publast = 0})
+end
+
+-- collect output (assume only one call to puts)
+r.puts = function(r, ...) _CACHE.reply = JSON.decode(...) end
+
+-- TODO
+r.escape_html = function(r, val)
+ -- < > & are definitely escaped by the real escape_html
+ return val:gsub('>', '>'):gsub('<', '<'):gsub('&', '&')
+end
+
+-- override the parse-args function so it returns our test data
+r.parseargs = function(r)
+ return _CACHE.args
+end
+
+
+local function test(args)
+ local output = {
+ quick = true, -- disable most queries
+ }
+ -- merge in user data
+ for k,v in pairs(args) do output[k] = v end
+ _CACHE.args = output -- save the args
+ _CACHE.status = handle(r)
+ return _CACHE
+end
+
+local argc = #arg
+if argc == 0 -- assume reading lines of JSON strings
+then
+ for line in io.lines()
+ do
+ jzon = JSON.decode(line)
+ jzon.quick = true -- disable most queries
+ _CACHE.args = jzon
+ _CACHE.status = handle(r)
+ print(JSON.encode(_CACHE))
+ io.flush()
+ end
+elseif argc % 2 == 0
+then
+ local data = {}
+ for i = 1,argc,2 do
+ data[arg[i]] = arg[i+1]
+ end
+ res = test(data)
+ if os.getenv("MODE") == "inspect" then
+ print(inspect(res["querydata"]))
+ else
+ print(JSON.encode(res))
+ end
+elseif argc == 1 -- assume JSON string
+then
+ jzon = JSON.decode(arg[1])
+ jzon.quick = true -- disable most queries
+ _CACHE.args = jzon
+ _CACHE.status = handle(r)
+ print(JSON.encode(_CACHE))
+else
+ print("Need even arg count")
+ os.exit(1)
+end
diff --git a/tools/import-mbox.py b/tools/import-mbox.py
index 28c0363..59f94f8 100755
--- a/tools/import-mbox.py
+++ b/tools/import-mbox.py
@@ -161,7 +161,7 @@
bmd = bf.read()
bf.close() # explicit early close
bmd = gzip.decompress(bmd)
- tmpfile = tempfile.NamedTemporaryFile(mode='w+b', buffering=1, delete=False)
+ tmpfile = tempfile.NamedTemporaryFile(mode='w+b', delete=False)
tmpfile.write(bmd)
tmpfile.flush()
tmpfile.close()
@@ -514,7 +514,7 @@
for mlist in re.finditer(ns, data):
ml = mlist.group(1)
mldata = urlopen("%s%s" % (source, ml)).read()
- tmpfile = tempfile.NamedTemporaryFile(mode='w+b', buffering=1, delete=False)
+ tmpfile = tempfile.NamedTemporaryFile(mode='w+b', delete=False)
try:
if ml.find(".gz") != -1:
mldata = gzip.decompress(mldata)
