#!/usr/bin/env ruby

=begin
Checks a mirror URL for compliance with ASF mirroring guidelines.

Derived partly from
https://svn.apache.org/repos/asf/infrastructure/site-tools/trunk/mirrors/check_mirror.pl

TODO this is a work in progress...

Ideally the causes of some of the problems should be reported ...

Note: the GUI interface is currently at www/members/mirror_check.cgi

=end

require 'wunderbar'
require 'net/http'

=begin
Checks performed: (F=fatal, E=error, W=warn)
F: zzz/time.txt is readable
F: its contents is a number followed by text
W: test whether time is more than 1 day old
W: test its content-type (missing or text/plain)
F: BASE is readable and non-empty
W: has html + body headers and body/html trailers
W: body matches m!<(img|IMG) (src|SRC)="/icons/!
E: check index against TLP list m!> ?$dir/?<!
E: tlp dir: check can be read (mirrors sometimes have incorrect protections)
W: 'favicon.ico' and 'zzz/' must both be in page
W: favicon.ico must appear after zzz/ to show folders first
E: 'harmony' should be redirected with 404
E: 'zzz/___' should generate 404
W: 'zzz/README' content-type text/plain
E: header must match /<h\d>Apache Software Foundation Distribution Meta-Directory</h\d>/
E: footer must match /This directory contains meta-data for the ASF mirroring system./
E: mirror-tests/ must exist
W: its files must not have content-encoding:
   1mb.img.7z 1mb.img.bz2 1mb.img.tar.gz 1mb.img.tgz 1mb.img.zip
W: zzz/mirror-tests/redirect-test/ should redirect to http://www.apache.org/ (302)

TODO - any more checks?

=end

$SAFE = 1

URLMATCH = %r!^https?://[^/]+/(\S+/)?$!i
HTTPDIRS = %w(zzz/ zzz/mirror-tests/) # must exist
HDRMATCH = %r!<h\d>Apache Software Foundation Distribution Meta-Directory</h\d>! # must be on the zzz index page
FTRMATCH = %r!This directory contains meta-data for the ASF mirroring system.! # must be on the zzz index page
HASHDR =   %r!<html( [^>]+)?>.+?<body>!im
HASFTR =   %r!</body>.*?</html>!im

HTTPDIR = 'zzz/' # must appear in index page
HTTP404 = 'zzz/___'; # Non-existent URL; should generate 404
HTTPTEXT = 'zzz/README'; # text file (without extension) should generate Content-Type text/plain or none

MIRRORTEST = 'zzz/mirror-tests/';
MIRRORTEST_FILES = %w(1mb.img.7z 1mb.img.bz2 1mb.img.tar.gz 1mb.img.tgz 1mb.img.zip) # no Content-Encoding !

# save the result of a test
def test(severity, txt)
  @tests << {severity => txt}
  @fails+=1 unless severity == :I
end

def F(txt)
  test(:F, txt)
end

def E(txt)
  test(:E, txt)
end

def W(txt)
  test(:W, txt)
end

def I(txt)
  test(:I, txt)
end

# extract test entries with key k
def tests(k)
  @tests.map{|t| t[k]}.compact
end

# get an HTTP URL
def getHTTPHdrs(url)
  url.untaint
  uri = URI.parse(url)
  http = Net::HTTP.new(uri.host, uri.port)
  http.use_ssl = uri.scheme == 'https'
  request = Net::HTTP::Head.new(uri.request_uri)
  http.request(request)
end

def check_redirect(base, page, expectedLocation, severity=:W, expectedStatus = "302", log=true)
  path = base + page
  response = getHTTPHdrs(path)
  if response.code != expectedStatus
    test severity, "HTTP status #{response.code} for '#{path}'" unless severity == nil
    return nil
  end
  if response['location'] != expectedLocation
    test severity, "HTTP location #{response['location']} for '#{path}' - expected '#{expectedLocation}'" unless severity == nil
    return nil
  end
  I "Fetched #{path} - redirected OK to #{response['location']}" if log
  response
end

def check_CT(base, page, severity=:E, expectedStatus = "200")
  path = base + page
  response = getHTTPHdrs(path)
  if response.code != expectedStatus
    test severity, "HTTP status #{response.code} for '#{path}'" unless severity == nil
    return nil
  end
  ct = response['Content-Type'] || 'unknown'
  ce = response['Content-Encoding']
  # TODO also check CT - some mirrors return text/plain for img??
  if ce
    W "Checking #{path} - Content-Type: #{ct} WARN: Content-Encoding: #{ce}"
  else
    I "Checking #{path} - Content-Type: #{ct}"
  end
end

# get an HTTP URL=> response
def getHTTP(url)
  url.untaint
  uri = URI.parse(url)
  http = Net::HTTP.new(uri.host, uri.port)
  http.use_ssl = uri.scheme == 'https'
  request = Net::HTTP::Get.new(uri.request_uri)
  http.request(request)
end

# check page can be read => body
def check_page(base, page, severity=:E, expectedStatus="200", log=true)
  path = base + page
  response = getHTTP(path)
  code = response.code ||  '?'
  if code != expectedStatus
    test(severity, "Fetched #{path} - HTTP status: #{code} expected: #{expectedStatus}") unless severity == nil
    return nil
  end
  I "Fetched #{path} - OK" if log
  response.body
end

def checkIndex(page, type)
  asfData = @pages[type]
  links = parseIndexPage(page)
  if type == :tlps
    fav = links.index('favicon.ico')
    zzz = links.index('zzz')
    if fav and zzz
      if fav < zzz
        W "Index for #{type}: incorrect #{type} page order - found favicon.ico before zzz/; folders should be listed before files"
      else
        I "Index for #{type}: found favicon.ico and zzz/ in the page in the correct order (i.e. folders are listed before files)"
      end
    else
      W "Index for #{type}: expected to find favicon.ico #{fav} and zzz/ #{zzz} in the page, but at least one is missing"
    end
  end
  links.each {|l|
    W "Index for #{type}: the link #{l} is not shown on ASF site" unless asfData.include? l
  }
  asfData.each {|l|
    W "Index for #{type}: the link #{l} is not shown on the mirror site" unless links.include? l or l == 'openoffice'
  }
end

# nginx <tr><td><a href="activemq/" title="activemq">activemq/</a></td><td>-</td><td>2019-Nov-25 18:00</td></tr>
# ASF <tr><td valign="top"><img src="/icons/folder.gif" alt="[DIR]"></td><td><a href="accumulo/">accumulo/</a></td><td align="right">2019-08-07 23:42  </td><td align="right">  - </td><td>&nbsp;</td></tr>


# parse an HTTP server Index page => array of file/folder names
def parseIndexPage(page)
  folders = []
  # ASF main page references currently look like this: <a href="abdera/">abdera/</a>
  # the Perl script looked for this match: m!> ?$dir/?<!
  links = page.scan(%r{<a href=['"]([.a-z0-9-]+)/?['"](?: title=['"][.a-z0-9-]+/?['"])?>([.a-z0-9-]+)/?</a>})
  links.each { |l|
    if l[1] == l[0]
      folders << l[1]
    end
  }
  folders
end
# Check page has sensible headers and footers
def checkHdrFtr(path, body)
  hasHTMLhdr = HASHDR.match(body)
  hasHTMLftr = HASFTR.match(body)
  if hasHTMLhdr
    if hasHTMLftr
      I "#{path} has header and footer"
    else
      W "#{path} is incomplete - no footer found"
    end
  else # no header
    if hasHTMLftr
      W "#{path} is incomplete - no header found"
    else
      W "#{path} is incomplete - no header or footer found"
    end
  end
end

# Suite: perform all the HTTP checks
def checkHTTP(base)
  # We don't check the pattern on the form for two reasons:
  # - not all browsers support it
  # - allows the input to be more flexible

  # Fix up the URL
  base.strip!
  base += '/' unless base.end_with? '/'
  base = 'http://' + base unless base.start_with? 'http'
  # Now check the syntax:

  I "Checking #{base} ..."

  unless URLMATCH.match(base)
    F "Invalid URL syntax: #{base}"
    return
  end

  setup

  response = getHTTPHdrs(base)
  server = response['server']
  if server =~ /Apache/
    I "Server: #{server}"
  else
    W "Server: '#{server}' - expected 'Apache' in server response"
  end

  # Check the mirror time (and that zzz/ is readable)
  time = check_page(base, 'zzz/time.txt', severity = :F)
  if time
    match = /^(\d+) \S+$/.match(time)
    if match
      now = Time.now.to_i
      stamp = match[1].to_i
      age = (now - stamp)/60 # minutes
      if age > 60*24
        W "Mirror is over 1 day old: #{age} minutes"
      else
        I "Mirror is less than 1 day old: #{age} minutes"
      end
    else
      F "Invalid time.txt contents: #{time}"
    end
  else
    return # cannot process further (already recorded the error
  end

  # check the main body
  body = check_page(base, '')
  checkHdrFtr(base, body)
  if %r{<(img|IMG) (src|SRC)="/icons/}.match(body)
    I "Index page has icons as expected"
  else
    W "Missing or unexpected img icon tags"
  end
  checkIndex(body, :tlps)

  ibody = check_page(base, 'incubator/')
  checkHdrFtr(base+'incubator/', ibody)
  checkIndex(ibody, :podlings)

  check_page(base, 'harmony/', :E, expectedStatus="404")

  zbody = check_page(base, HTTPDIR)
# Not sure this is useful on its own anymore
# It was originally used to detect sites with advertising wrappers,
# but most recent examples have been tables around directory listings
# which is obviously OK as it does not affect the user experience.
#  if  %r{<table}i.match(zbody)
#    W "#{HTTPDIR} - TABLE detected"
#  else
#    I "#{HTTPDIR} - No TABLE detected, OK"
#  end
  checkHdrFtr(base+HTTPDIR, zbody)
  if HDRMATCH.match(zbody)
    I "Index page for #{HTTPDIR} contains the expected header text"
  else
    W "Index page for #{HTTPDIR} does not contain the expected header text"
  end
  if FTRMATCH.match(zbody)
    I "Index page for #{HTTPDIR} contains the expected footer text"
  else
    W "Index page for #{HTTPDIR} does not contain the expected footer text"
  end

  check_page(base,HTTP404,:E, expectedStatus="404")

  # Check that archives don't have Content-Encoding
  MIRRORTEST_FILES.each do |file|
    check_CT(base, MIRRORTEST + file)
  end
  check_redirect(base, 'zzz/mirror-tests/redirect-test/xyz', 'http://www.apache.org/')
end

def init
  # build a list of validation errors
  @tests = []
  @fails = 0
end

def setup
  tlps = parseIndexPage(check_page('https://downloads.apache.org/','',:F,"200",log=false))
  podlings = parseIndexPage(check_page('https://downloads.apache.org/incubator/','',:F,"200",false))
  @pages = {:tlps => tlps, :podlings => podlings}
end

def showList(list, header)
  unless list.empty?
    _h2_ header
    _ul do
      list.each { |item| _li item }
    end
  end
end

def display
  fatals = tests(:F)
  errors = tests(:E)
  warns = tests(:W)

  if !fatals.empty?
    _h2_.bg_danger "The mirror at #@url failed our checks:"
  elsif !errors.empty?
    _h2_.bg_warning "The mirror at #@url has some problems:"
  elsif !warns.empty?
    _h2_.bg_warning "The mirror at #@url has some minor issues"
  else
    _h2_.bg_success "The mirror at #@url looks OK, thanks for using this service"
  end

  if @fails > 0
    showList(fatals, "Fatal errors:")
    showList(errors, "Errors:")
    showList(warns, "Warnings:")
    # Cannot easily copy/paste URLs; use layout suitable for copy/paste into e.g. JIRA issue/e-mail
    _p do
      _ 'Please see the Apache mirror configuration instructions [1] for further details on configuring your mirror server.'
    end
    _p do
    _ '[1] '
      _a 'http://www.apache.org/info/how-to-mirror.html#Configuration', href: 'http://www.apache.org/info/how-to-mirror.html#Configuration'
    end
  end

  _h2_ 'Tests performed'
  _ol do
    @tests.each { |t| t.map{|k,v| _li "#{k}: - #{v}"}}
  end
  _h4_ 'F: fatal, E: Error, W: warning, I: info (success)'
end

# Called by GUI when POST is pushed
def doPost(url)
  init
  checkHTTP(url)
  display
end

if __FILE__ == $0
  $SAFE = 0
  init
  url = ARGV[0] || "localhost" # easier to test in an IDE
  checkHTTP(url+"") # allow url to be untainted later
  # display the test results
  @tests.each { |t| t.map{|k, v| puts "#{k}: - #{v}"}}
  if @fails > 0
    puts "#{url} had #{@fails} errors"
  else
    puts "#{url} passed all the tests"
  end
end