Merge branch 'master' into gs/update-proxies
diff --git a/tools/mboxhdr2csv.rb b/tools/mboxhdr2csv.rb
index 1ebd520..c89ddd0 100644
--- a/tools/mboxhdr2csv.rb
+++ b/tools/mboxhdr2csv.rb
@@ -156,64 +156,67 @@
# Return cached calculated data if present
cache_json = File.join(mailroot, "#{yearmonth}.json")
if File.file?(cache_json)
- return JSON.parse(File.read(cache_json))
- else
- emails = {}
- files = Dir[File.join(mailroot, yearmonth, '*')]
- return emails if files.empty?
- emails[MAILS] = []
- emails[TOOLS] = []
- files.each do |email|
- next if email.end_with? '/index'
- message = IO.read(email.untaint, mode: 'rb')
- data = {}
- data[DATE] = DateTime.parse(message[/^Date: (.*)/, 1]).iso8601
- data[FROM] = message[/^From: (.*)/, 1]
- # Originally (before 2265343) the local method #find_who_from expected an email address and returned who, committer
- # Emulate this with the version from MailUtils which expects and updates a hash
- temp = {from: data[FROM]} # pass a hash
- MailUtils.find_who_from(temp) # update the hash
- # pick out the bits we want
- data[WHO], data[COMMITTER] = temp[:who], temp[:committer]
-
- data[SUBJECT] = message[/^Subject: (.*)/, 1]
- if nondiscuss
- nondiscuss.each do |typ, rx|
- if data[SUBJECT] =~ rx
- data[TOOLS] = typ
- break # regex.each
- end
- end
- end
- data.has_key?(TOOLS) ? emails[TOOLS] << data : emails[MAILS] << data
+ begin
+ return JSON.parse(File.read(cache_json))
+ rescue StandardError => e
+ # No-op: fall through to attempt to re-create cache
end
- # Provide as sorted data for ease of use
- emails[TOOLS].sort_by! { |email| email[DATE] }
- emails[TOOLCOUNT] = Hash.new {|h, k| h[k] = 0 }
- emails[TOOLS].each do |mail|
- emails[TOOLCOUNT][mail[TOOLS]] += 1
- end
- emails[TOOLCOUNT] = emails[TOOLCOUNT].sort_by { |k,v| -v}.to_h
-
- emails[MAILS].sort_by! { |email| email[DATE] }
- emails[MAILCOUNT] = Hash.new {|h, k| h[k] = 0 }
- emails[MAILS].each do |mail|
- emails[MAILCOUNT][mail[WHO]] += 1
- end
- emails[MAILCOUNT] = emails[MAILCOUNT].sort_by { |k,v| -v}.to_h
-
- # If yearmonth is before current month, then write out yearmonth.json as cache
- if yearmonth < Date.today.strftime('%Y%m')
- begin
- File.open(cache_json, 'w') do |f|
- f.puts JSON.pretty_generate(emails)
- end
- rescue
- # No-op, just don't cache for now
- end
- end
- return emails
end
+ emails = {}
+ files = Dir[File.join(mailroot, yearmonth, '*')]
+ return emails if files.empty?
+ emails[MAILS] = []
+ emails[TOOLS] = []
+ files.each do |email|
+ next if email.end_with? '/index'
+ message = IO.read(email.untaint, mode: 'rb')
+ data = {}
+ data[DATE] = DateTime.parse(message[/^Date: (.*)/, 1]).iso8601
+ data[FROM] = message[/^From: (.*)/, 1]
+ # Originally (before 2265343) the local method #find_who_from expected an email address and returned who, committer
+ # Emulate this with the version from MailUtils which expects and updates a hash
+ temp = {from: data[FROM]} # pass a hash
+ MailUtils.find_who_from(temp) # update the hash
+ # pick out the bits we want
+ data[WHO], data[COMMITTER] = temp[:who], temp[:committer]
+
+ data[SUBJECT] = message[/^Subject: (.*)/, 1]
+ if nondiscuss
+ nondiscuss.each do |typ, rx|
+ if data[SUBJECT] =~ rx
+ data[TOOLS] = typ
+ break # regex.each
+ end
+ end
+ end
+ data.has_key?(TOOLS) ? emails[TOOLS] << data : emails[MAILS] << data
+ end
+ # Provide as sorted data for ease of use
+ emails[TOOLS].sort_by! { |email| email[DATE] }
+ emails[TOOLCOUNT] = Hash.new {|h, k| h[k] = 0 }
+ emails[TOOLS].each do |mail|
+ emails[TOOLCOUNT][mail[TOOLS]] += 1
+ end
+ emails[TOOLCOUNT] = emails[TOOLCOUNT].sort_by { |k,v| -v}.to_h
+
+ emails[MAILS].sort_by! { |email| email[DATE] }
+ emails[MAILCOUNT] = Hash.new {|h, k| h[k] = 0 }
+ emails[MAILS].each do |mail|
+ emails[MAILCOUNT][mail[WHO]] += 1
+ end
+ emails[MAILCOUNT] = emails[MAILCOUNT].sort_by { |k,v| -v}.to_h
+
+ # If yearmonth is before current month, then write out yearmonth.json as cache
+ if yearmonth < Date.today.strftime('%Y%m')
+ begin
+ File.open(cache_json, 'w') do |f|
+ f.puts JSON.pretty_generate(emails)
+ end
+ rescue
+ # No-op, just don't cache for now
+ end
+ end
+ return emails
end
end
diff --git a/www/index.html b/www/index.html
index 850fc09..ba69ee8 100644
--- a/www/index.html
+++ b/www/index.html
@@ -46,6 +46,11 @@
</a>
</li>
<li role="presentation">
+ <a href="https://issues.apache.org/jira/projects/WHIMSY/issues">
+ Issues
+ </a>
+ </li>
+ <li role="presentation">
<a href="status/">
Server status
</a>
@@ -83,6 +88,7 @@
Our focus is on providing organizational information about the ASF and our projects
in easy to consume ways, and to help automate corporate processes at the ASF to
make the paperwork behind the scenes easier for our many volunteers.
+ Please see the Code, Questions, or Issues links above to learn more about the project.
</p>
</div>
</div>
diff --git a/www/members/list-traffic.cgi b/www/members/list-traffic.cgi
index 908e772..7e32d65 100755
--- a/www/members/list-traffic.cgi
+++ b/www/members/list-traffic.cgi
@@ -32,11 +32,11 @@
months.sort.reverse.each do |month|
data = MailUtils.get_mails_month(mailroot: SRV_MAIL, yearmonth: month, nondiscuss: nondiscuss)
next if data.empty?
- _h1 "#{LIST_ROOT}@ statistics for #{month} (total mails: #{data[MailUtils::MAILS].length + data[MailUtils::TOOLS].length})", id: "#{month}"
+ _h1 "#{LIST_ROOT}@ statistics for #{month} (total mails: #{data[MailUtils::MAILS].length})", id: "#{month}"
_div.row do
_div.col_sm_6 do
_ul.list_group do
- _li.list_group_item.active.list_group_item_info "Top Ten Email Senders (from non-tool mails: #{data[MailUtils::MAILS].length})"
+ _li.list_group_item.active.list_group_item_info "Top Ten Email Senders"
ctr = 0
data[MailUtils::MAILCOUNT].each do |id, num|
if num > (data[MailUtils::MAILS].length / 10)
@@ -51,9 +51,11 @@
end
_div.col_sm_6 do
_ul.list_group do
- _li.list_group_item.list_group_item_info "Tool Generated Emails (by type, total tool mails: #{data[MailUtils::TOOLS].length})"
- data[MailUtils::TOOLCOUNT].each do |id, num|
- _li.list_group_item "#{num} emails from #{id} tool"
+ _li.list_group_item.list_group_item_info "Long Tail - All Senders"
+ _li.list_group_item do
+ data[MailUtils::MAILCOUNT].each do |id, num|
+ _! "#{id} (#{num}), "
+ end
end
end
end
@@ -79,7 +81,7 @@
end
end
end
- _h1 "#{LIST_ROOT}@ list non-tool emails weekly statistics", id: "top"
+ _h1 "#{LIST_ROOT}@ list emails weekly statistics", id: "top"
_div.row do
_div.col.col_sm_offset_1.col_sm_9 do
weeks.sort.reverse.each do |week, senders|
@@ -118,29 +120,29 @@
_whimsy_body(
title: PAGETITLE,
related: {
- "/board/agenda" => "Current Month Board Agenda",
- "/board/minutes" => "Past Minutes, Categorized",
- "https://www.apache.org/foundation/board/calendar.html" => "Past Minutes, Dated",
- "#{ENV['SCRIPT_NAME']}" => "List Traffic By Month",
- "#{ENV['SCRIPT_NAME']}?week" => "List Traffic By Week",
+ "/members/index" => "More Member-Specific Tools",
+ "/officers/list-traffic" => "Board@ List Traffic",
+ "#{ENV['SCRIPT_NAME']}" => "Members@ List Traffic By Month",
+ "#{ENV['SCRIPT_NAME']}?week" => "Members@ List Traffic By Week",
"https://github.com/apache/whimsy/blob/master/www#{ENV['SCRIPT_NAME']}" => "See This Source Code"
},
helpblock: -> {
_p %{
- This script displays some simple (and potentially lossy) analysis of traffic on the #{LIST_ROOT}@ mailing list.
- In particular, mapping email to a committer may not work (meaning individual senders may have multiple spots),
+ This script displays simple (and likely slightly lossy) analysis of traffic on the #{LIST_ROOT}@ mailing list.
+ In particular, mapping From: email to a committer may not work (meaning individual senders may have multiple spots),
and Subject lines displayed may be truncated (meaning threads may not fully be tracked). Work in progress.
}
_p do
- _ 'This attempts to differentiate tool- or process-generated emails (NOTICE, REPORT, etc.) from all other emails (i.e. mails hand-written by a person). '
- _ 'Senders of more than 10% of all non-tool emails in a month are highlighted. '
- _ 'Senders of more than 20%, 10%, or 5% of all non-tool emails in a week are highlighted in the '
+ _ 'Senders of more than 10% of all emails in a month are highlighted. '
+ _ 'Senders of more than 20%, 10%, or 5% of all emails in a week are highlighted in the '
_a 'By week view (supply ?week in URL).', href: '?week'
end
}
) do
months = Dir["#{SRV_MAIL}/*"].map {|path| File.basename(path).untaint}.grep(/^\d+$/)
+ _.error "HACK - server log one"
+
if ENV['QUERY_STRING'].include? 'week'
display_weekly(months: months, nondiscuss: MailUtils::NONDISCUSSION_SUBJECTS["<#{LIST_ROOT}.apache.org>"])
else
diff --git a/www/members/proxy.cgi b/www/members/proxy.cgi
index 959ac25..7c2c8e3 100755
--- a/www/members/proxy.cgi
+++ b/www/members/proxy.cgi
@@ -76,7 +76,7 @@
end
end
else
- _p 'The following members have volunteered to serve as proxies; you can freely select any one of them below:'
+ _p 'The following members have explicitly volunteered to serve as proxies; select any one of them, or select any other member that you know will proxy for you (or ask!):'
_ul do
volunteers.each do |vol|
_pre vol
diff --git a/www/members/subscriptions.cgi b/www/members/subscriptions.cgi
index f263248..efd9779 100755
--- a/www/members/subscriptions.cgi
+++ b/www/members/subscriptions.cgi
@@ -51,6 +51,8 @@
_ 'Separate tables below show '
_a 'Members not subscribed to the list', href: "#unsub"
_ ', and '
+ _a 'Copyable list of Members not subscribed', href: "#unsublist"
+ _ ', and '
_a 'Entries in LDAP but not members.txt', href: "#ldap"
_ '.'
end
@@ -131,6 +133,12 @@
end
end
end
+ _h3_.unsublist! 'Handy List of Unsubscribed Emails'
+ _p do
+ missing.each do |person|
+ _ "#{person.id}@apache.org, "
+ end
+ end
end
extras = ldap - members
