#!/usr/bin/env ruby

=begin
Checks a download page URL for compliance with ASF guidelines.


Note: the GUI interface is currently at www/members/download_check.cgi

=end

$LOAD_PATH.unshift '/srv/whimsy/lib'
require 'whimsy/asf'
require 'wunderbar'
require 'net/http'
require 'nokogiri'
require 'time'

=begin
Checks performed: (F=fatal, E=error, W=warn)
TBA
=end

$SAFE = 1

$CLI = false
$VERBOSE = false

$ARCHIVE_CHECK = false
$ALWAYS_CHECK_LINKS = false
$NO_CHECK_LINKS = false
$NOFOLLOW = false # may be reset
$ALLOW_HTTP = false # http links generate Warning, not Error

$VERSION = nil

# match an artifact
# TODO detect artifacts by URL as well if possible
ARTIFACT_RE = %r{/([^/]+\.(tar|tar\.gz|deb|nbm|zip|tgz|tar\.bz2|jar|war|msi|exe|rar|rpm|nar))(&action=download)?$}

def init
  # build a list of validation errors
  @tests = []
  @fails = 0
  if $NO_CHECK_LINKS
    $NOFOLLOW = true
    I "Will not check links"
  else
    if $ALWAYS_CHECK_LINKS
      I "Will check links even if download page has errors"
    else
      I "Will check links if download page has no errors"
    end      
  end
  I "Will %s archive.apache.org links in checks" % ($ARCHIVE_CHECK ? 'include' : 'not include')
end

# save the result of a test
def test(severity, txt)
  @tests << {severity => txt}
  @fails +=1 unless severity == :I or severity == :W
end

def F(txt)
  test(:F, txt)
end

def E(txt)
  test(:E, txt)
end

def W(txt)
  test(:W, txt)
end

def I(txt)
  test(:I, txt)
end

# extract test entries with key k
def tests(k)
  @tests.map{|t| t[k]}.compact
end

# extract test entries with key k
def testentries(k)
  @tests.select{|t| t[k]}.compact
end

def showList(list, header)
  unless list.empty?
    _h2_ header
    _ul do
      list.each { |item| _li item }
    end
  end
end

def displayHTML
  fatals = tests(:F)
  errors = tests(:E)
  warns = tests(:W)

  if !fatals.empty?
    _h2_.bg_danger "The page at #@url failed our checks:"
  elsif !errors.empty?
    _h2_.bg_warning "The page at #@url has some problems:"
  elsif !warns.empty?
    _h2_.bg_warning "The page at #@url has some minor issues"
  else
    _h2_.bg_success "The page at #@url looks OK, thanks for using this service"
  end

  if @fails > 0
    showList(fatals, "Fatal errors:")
    showList(errors, "Errors:")
  end

  showList(warns, "Warnings:")

  _h2_ 'Tests performed'
  _ol do
    @tests.each { |t| t.map{|k,v| _li "#{k}: - #{v}"}}
  end
  _h4_ 'F: fatal, E: Error, W: warning, I: info (success)'
end

# get an HTTP URL
def HEAD(url)
  puts ">> HEAD #{url}" if $VERBOSE
  url.untaint
  uri = URI.parse(url)
  unless uri.scheme
    W "No scheme for URL #{url}, assuming http"
    uri = URI.parse("http:"+url)
  end
  http = Net::HTTP.new(uri.host, uri.port)
  http.use_ssl = uri.scheme == 'https'
  request = Net::HTTP::Head.new(uri.request_uri)
  http.request(request)
end

# get an HTTP URL=> response
def GET(url)
  puts ">> GET #{url}" if $VERBOSE
  url.untaint
  uri = URI.parse(url).untaint
  unless uri.scheme
    W "No scheme for URL #{url}, assuming http"
    uri = URI.parse("http:"+url).untaint
  end
  http = Net::HTTP.new(uri.host, uri.port)
  http.use_ssl = uri.scheme == 'https'
  request = Net::HTTP::Get.new(uri.request_uri)
  http.request(request.untaint)
end

# Check page exists
def check_head(path, severity = :E, expectedStatus = "200", log=true, allowRedirect=false)
  response = HEAD(path)
  code = response.code ||  '?'
  if code == '403' # someone does not like Whimsy?
    W "HEAD #{path} - HTTP status: #{code} - retry"
    response = HEAD(path)
    code = response.code ||  '?'
  elsif allowRedirect and code == '302'
    location = response['location']
    if location.start_with? 'https://downloads.apache.org/'
      response = HEAD(location)
      code = response.code ||  '?'
    end
  end
  if code != expectedStatus
    test(severity, "HEAD #{path} - HTTP status: #{code} expected: #{expectedStatus}") unless severity == nil
    return nil
  end
  I "Checked HEAD #{path} - OK (#{code})" if log
  response
end

# check page can be read => body
def check_page(path, severity=:E, expectedStatus="200", log=true)
  response = GET(path)
  code = response.code ||  '?'
  if code != expectedStatus
    test(severity, "Fetched #{path} - HTTP status: #{code} expected: #{expectedStatus}") unless severity == nil
    return nil
  end
  I "Fetched #{path} - OK (#{code})" if log
  puts "Fetched #{path} - OK (#{code})" if $CLI
  if code == '200'
    return response.body
  else
    return response
  end
end

# Check closer/download page
def check_closer_down(url)
  # N.B. HEAD does not work; it returns success
  res = check_page(url, :E, "302", false)
  loc = res['location']
  res = check_head(loc, :E, "200", false)
  return unless res
  ct = res.content_type
  cl = res.content_length
  if ct and cl
    I "Checked #{url} OK - ct=#{ct} cl=#{cl}"
  elsif cl > 0
    W "Possible issue with #{url} ct=#{ct} cl=#{cl}"
  else
    E "Problem with #{url} ct=#{ct} cl=#{cl}"
  end
end

def WE(msg)
  if $ALLOW_HTTP
    W msg
  else
    E msg
  end
end
# returns www|archive, stem and the hash extension
def check_hash_loc(h,tlp)
  if h =~ %r{^(https?)://(?:(archive|www)\.)?apache\.org/dist/(?:incubator/)?#{tlp}/.*?([^/]+)\.(\w{3,6})$}
    WE "HTTPS! #{h}" unless $1 == 'https'
    return $2,$3,$4
  elsif h =~ %r{^(https?)://(downloads)\.apache\.org/(?:incubator/)?#{tlp}/.*?([^/]+)\.(\w{3,6})$}
    WE "HTTPS! #{h}" unless $1 == 'https'
    return $2,$3,$4
  elsif h =~ %r{^(https?)://repo\.(maven)\.apache\.org/maven2/org/apache/#{tlp}/.+/([^/]+)(\.(\w{3,6}))$} # Maven
    WE "HTTPS! #{h}" unless $1 == 'https'
    W "Unexpected hash location #{h} for #{tlp}"
    return $2,$3,$4
  else
    E "Unexpected hash location #{h} for #{tlp}"
    nil
  end
end

# get the https? links as Array of [href,text]
def get_links(body)
  doc = Nokogiri::HTML(body)
  nodeset = doc.css('a[href]')    # Get anchors w href attribute via css
  links = nodeset.map {|node|
    href = node.attribute("href").to_s
    text = node.text.gsub(/[[:space:]]+/,' ')
    [href,text]
  }.select{|x,y| x =~ %r{^(https?:)?//} }
end

VERIFY_TEXT = [
 'the integrity of the downloaded files',
 'verify the integrity', # commons has this as a link; perhaps try converting page to text only?
 'verify that checksums and signatures are correct',
 '#verifying-signature',
 'check that the download has completed OK',
 'You should verify your download',
 'downloads can be verified',
 'www.apache.org/info/verification.html',
 'verify your mirrored downloads',
 'verify your downloads',
 'verify the downloaded files',
 'All downloads should be verified',
 'verification instructions',
 ' encouraged to verify ',
 'To check a GPG signature',
]

ALIASES = {
    'sig' => 'asc',
    'pgp' => 'asc',
    'signature' => 'asc',
    'ascsignature' => 'asc',
    'pgpsignature' => 'asc',
    'pgpsignatures' => 'asc',
    'gpgsignature' => 'asc',
    'openpgpsignature' => 'asc',
}

# Need to be able to check if download is for a PMC or podling
# parameter is the website URL
# Also want to convert site to TLP
URL2TLP = Hash.new # URL host to TLP conversion
URL2TLP['jspwiki-wiki'] = 'jspwiki' # https://jspwiki-wiki.apache.org/Wiki.jsp?page=Downloads
PMCS = Set.new # is this a TLP?
ASF::Committee.pmcs.map do |p|
    site = p.site[%r{//(.+?)\.apache\.org},1]
    name = p.name
    URL2TLP[site] = name unless site == name
    PMCS << name
end

# Convert text reference to extension
# e.g. SHA256 => sha256; [SIG] => asc
def text2ext(txt)
    # need to strip twice to handle ' [ asc ] '
    # TODO: perhaps just remove all white-space?
    tmp = txt.downcase.strip.sub(%r{^\[(.+)\]$},'\1').sub('-','').sub(/ ?(digest|checksum)/,'').sub(/ \(tar\.gz\)| \(zip\)| /,'').strip
    ALIASES[tmp] || tmp
end

# Suite: perform all the HTTP checks
def checkDownloadPage(path, tlp, version)
    begin
        _checkDownloadPage(path.strip, tlp, version)
    rescue Exception => e
        F e
        if $CLI
          p e
          puts e.backtrace 
        end
    end
end

def _checkDownloadPage(path, tlp, version)
  isTLP = PMCS.include? tlp
  if version != ''
    I "Checking #{path} [#{tlp}] TLP #{isTLP} for version #{version} only ..."
  else
    I "Checking #{path} [#{tlp}] TLP #{isTLP} ..."
  end

  # check the main body
  if path.start_with? 'http'
    body = check_page(path)
  else
    file = path
    if file.start_with? '~'
      file = ENV['HOME'] + file[1..-1]
    end
    body = File.read(file.untaint)
  end
  
  return unless body

  hasDisclaimer = body.gsub(%r{\s+},' ').include? 'Incubation is required of all newly accepted'

  if isTLP
     W "#{tlp} has Incubator disclaimer" if hasDisclaimer
  else
     if hasDisclaimer
       I "#{tlp} has Incubator disclaimer"
     else
       E "#{tlp} does not have Incubator disclaimer"
     end
  end

  if body.include? 'dist.apache.org'
    E 'Page must not link to dist.apache.org'
  else
    I 'Page does not reference dist.apache.org'
  end

  if body.include? 'repository.apache.org'
    E 'Page must not link to repository.apache.org'
  else
    I 'Page does not reference repository.apache.org'
  end

  deprecated = Time.parse('2018-01-01')
  
  links = get_links(body)
  
  # check KEYS link
  # TODO: is location used by hc allowed, e.g.
  #   https://www.apache.org/dist/httpcomponents/httpclient/KEYS
  expurl = "https://[downloads.|www.]apache.org/[dist/][incubator/]#{tlp}/KEYS"
  expurlre = %r{^https://((www\.)?apache\.org/dist|downloads\.apache\.org)/(incubator/)?#{tlp}/KEYS$}
  keys = links.select{|h,v| h =~ expurlre}
  if keys.size >= 1
    keyurl = keys.first.first
    keytext = keys.first[1]
    if keytext.strip == 'KEYS'
        I 'Found KEYS link'
    else
        W "Found KEYS: '#{keytext}'"
    end
    check_head(keyurl,:E, "200", false, true)
  else
    keys = links.select{|h,v| v.strip == 'KEYS' || v == 'KEYS file' || v == '[KEYS]'}
    if keys.size >= 1
      I 'Found KEYS link'
      keyurl = keys.first.first
      if keyurl =~ expurlre
        I "KEYS links to #{expurl} as expected"
      else
        if keyurl =~ %r{^https://www\.apache\.org/dist/#{tlp}/[^/]+/KEYS$}
          W "KEYS: expected: #{expurl}\n             actual: #{keyurl}"
        elsif keyurl =~ %r{^https://downloads\.apache\.org/#{tlp}/[^/]+/KEYS$}
          W "KEYS: expected: #{expurl}\n             actual: #{keyurl}"
        else
          E "KEYS: expected: #{expurl}\n             actual: #{keyurl}"
        end
      end
      check_head(keyurl,:E, "200")
    else
      E 'Could not find KEYS link'
    end
  end
  
  # check for verify instructions
  bodytext = body.gsub(/\s+/,' ') # single line
  if VERIFY_TEXT.any? {|text| bodytext.include? text}
    I 'Found reference to download verification'
  else
    E 'Could not find statement of the need to verify downloads'
  end
  
  # Check if GPG verify has two parameters
  body.scan(%r{^.+gpg --verify.+$}){|m|
    unless m =~ %r{gpg --verify\s+\S+\.asc\s+\S+}
      W "gpg verify should specify second param: #{m.strip} see:\nhttps://www.apache.org/info/verification.html#specify_both"
    end
  }
  
  # check if page refers to md5sum
  body.scan(%r{^.+md5sum.+$}){|m|
    W "Found md5sum: #{m.strip}"
  }
  
  # Check archives have hash and sig
  vercheck = Hash.new() # key = archive name, value = array of hash/sig

  links.each do |h,t|
    # These might also be direct links to mirrors
    if h =~ ARTIFACT_RE
        base = File.basename($1)
  #         puts "base: " + base
        if vercheck[base]  # might be two links to same archive
            W "Already seen link for #{base}"
        else
            vercheck[base] = [h =~ %r{^https?://archive.apache.org/} ? 'archive' : 'live']
        end
        # Text must include a '.' (So we don't check 'Source')
        if t.include?('.') and not base == File.basename(t.strip.downcase)
          # text might be short version of link
          tmp = t.strip.sub(%r{.*/},'').downcase # 
          if base == tmp
            W "Mismatch?: #{h} and '#{t}'"
          elsif base.end_with? tmp
            W "Mismatch?: #{h} and '#{tmp}'"
          elsif base.sub(/-bin\.|-src\./,'.').end_with? tmp
            W "Mismatch?: #{h} and '#{tmp}'"
          else
            W "Mismatch2: #{h} and '#{tmp}'"
          end
        end        
    end
  end

  links.each do |h,t|
    # Must occur before mirror check below
    if h =~ %r{^https?://(?:(?:archive\.|www\.)?apache\.org/dist|downloads\.apache\.org|repo\.maven\.apache\.org/maven2/.+?)/(.+\.(asc|sha\d+|md5|sha))$}
        base = File.basename($1)
        ext = $2
        stem = base[0..-(2+ext.length)]
        if vercheck[stem]
          vercheck[stem] << ext
        else
          E "Bug: found hash #{h} for missing artifact #{stem}"
        end
        t.strip!
        next if t == '' # empire-db
        tmp = text2ext(t)
        next if ext == tmp # i.e. link is just the type or [TYPE]
        next if ext == 'sha' and tmp == 'sha1' # historic
        next if (ext == 'sha256' or ext == 'sha512') and (t == 'SHA' or t == 'digest') # generic
        if not base == t and not t == 'checksum'
            E "Mismatch: #{h} and '#{t}'"
        end
    end
  end

  
  # did we find all required elements?
  vercheck.each do |k,v|
    typ = v.shift
    unless v.include? "asc" and v.any? {|e| e =~ /^sha\d+$/ or e == 'md5' or e == 'sha'}
      if typ == 'live'
        E "#{k} missing sig/hash: (found only: #{v.inspect})"
      elsif typ == 'archive'
        W "#{k} missing sig/hash: (found only: #{v.inspect})"
      else
        E "#{k} missing sig/hash: (found only: #{v.inspect}) TYPE=#{typ}"
      end
    end
  end

  if @fails > 0 and not $ALWAYS_CHECK_LINKS
    W "** Not checking links **"
    $NOFOLLOW = true
  end

  # Check if the links can be read

  links.each do |h,t|
    if h =~ %r{\.(asc|sha256|sha512)$}
      host, stem, ext = check_hash_loc(h,tlp)
      if host == 'archive'
        if $ARCHIVE_CHECK
          check_head(h, :E, "200", true, true)
        else
          I "Ignoring archive hash #{h}"
        end
      elsif host
        if $NOFOLLOW
          I "Skipping archive hash #{h}"
        else
          check_head(h, :E, "200", true, true)
        end
      else
        # will have been reported by check_hash_loc
      end
    # mirror downloads need to be treated differently
    elsif h =~ %r{^https?://(www\.)?apache\.org/dyn/.*action=download}
      if $NOFOLLOW
          I "Skipping download artifact #{h}"
      else
          check_closer_down(h)
      end
    elsif h =~ ARTIFACT_RE
      if $NOFOLLOW
        I "Skipping archive artifact #{h}"
        next
      end
      name = $1
      ext = $2
      if h =~ %r{https?://archive\.apache\.org/}
        unless $ARCHIVE_CHECK
            I "Ignoring archive artifact #{h}"
            next
        end
      end
      # Ideally would like to check for use of closer.lua/.cgi, but some projects pre-populate the pages
      # TODO: would it help to check host against mirrors.list?
      if h =~ %r{https?://(www\.)?apache\.org/dist} or h =~ %r{https?://downloads.apache.org/}
        E "Must use mirror system #{h}"
        next
      end
      res = check_head(h, :E, "200", false)
      next unless res
      # if HEAD returns content_type and length it's probably a direct link
      ct = res.content_type
      cl = res.content_length
      if ct and cl
        I "#{h} OK: #{ct} #{cl}"
      else # need to try to download the mirror page
        path = nil
        bdy = check_page(h, :E, "200", false)
        if bdy
          lks = get_links(bdy)
          lks.each do |l,t|
             # Don't want to match archive server (closer.cgi defaults to it if file is not found)
             if l.end_with?(name) and l !~ %r{//archive\.apache\.org/}
                path = l
                break
             end
          end
        end
        if path
          res = check_head(path, :E, "200", false)
          next unless res
          ct = res.content_type
          cl = res.content_length
          if ct and cl
            I "OK: #{ct} #{cl} #{path}"
          elsif cl
            W "NAK: ct='#{ct}' cl='#{cl}' #{path}"
          else
            E "NAK: ct='#{ct}' cl='#{cl}' #{path}"
          end
        else
          E "Could not find link for '#{name}' in page: '#{h}' (missing or archived)"
        end
      end
    elsif h =~ %r{\.(md5|sha\d*)$}
      host,_,_ = check_hash_loc(h,tlp)
      if $NOFOLLOW
        I "Skipping deprecated hash #{h}"
        next
      end
      if host == 'www' or host == ''
        res = check_head(h,:E, "200", false, true) # allow for redirect here
        next unless res
        lastmod = res['last-modified']
        date = Time.parse(lastmod)
        # Check if older than 2018?
        if date < deprecated
          I "Deprecated hash found #{h} #{t}; however #{lastmod} is older than #{deprecated}"
          # OK
        else
          W "Deprecated hash found #{h} #{t} - do not use for current releases #{lastmod}"
        end
      end
    elsif h =~ %r{/KEYS$} or t == 'KEYS'
      # already handled
    elsif h =~ %r{^https?://www\.apache\.org/?(licenses/.*|foundation/.*|events/.*)?$}
      # standard links
    elsif h =~ %r{https?://people.apache.org/phonebook.html}
    elsif h.start_with? 'https://cwiki.apache.org/confluence/'
      # Wiki
    elsif h.start_with? 'https://wiki.apache.org/'
      # Wiki
    elsif h.start_with? 'https://svn.apache.org/'
      #        E "Public download pages should not link to unreleased code: #{h}" # could be a sidebar/header link
    elsif h =~ %r{^https?://(archive|www)\.apache\.org/dist/}
      W "Not yet handled #{h} #{t}" unless h =~ /RELEASE[-_]NOTES/ or h =~ %r{^https?://archive.apache.org/dist/#{tlp}/}
    else
      # Ignore everything else?
    end
  end

end

def getTLP(url) # convert URL to TLP/podling
  if url =~ %r{^https?://cwiki\.apache\.org/confluence/display/([\S]+)/}
      tlp = $1.downcase
  elsif url =~ %r{^https?://([^.]+)(\.incubator|\.us|\.eu)?\.apache\.org/}
     tlp = URL2TLP[$1] || $1
  elsif url =~ %r{^https?://([^.]+)\.openoffice\.org/}
     tlp = 'openoffice'
  else
     tlp = nil
     F "Unknown TLP for URL #{url}"
  end
  tlp
end

# Called by GUI when POST is pushed
def doPost(options)
  $ALWAYS_CHECK_LINKS = options[:checklinks]
  $NO_CHECK_LINKS = options[:nochecklinks]
  $ARCHIVE_CHECK = options[:archivecheck]
  init
  url = options[:url]
  tlp = options[:tlp]
  tlp = getTLP(url) if tlp == ''
  if tlp
    checkDownloadPage(url, tlp, options[:version])
  end
  displayHTML
end


if __FILE__ == $0
  $CLI = true
  $SAFE = 0
  $VERBOSE =true
  $ALWAYS_CHECK_LINKS = ARGV.delete '--always'
  $NO_CHECK_LINKS = ARGV.delete '--nolinks'
  $ARCHIVE_CHECK = ARGV.delete '--archivecheck'
  $ALLOW_HTTP = ARGV.delete '--http'

  # check for any unhandled options
  ARGV.each do |arg|
    raise ArgumentError.new("Invalid option #{arg}") if arg.start_with? '--'
  end

  init

  version = ''
  if ARGV.size == 1
    url = ARGV[0]
    tlp = getTLP(url)
  else
    url = ARGV[0]
    tlp = ARGV[1]
    version = ARGV[2] || ''
  end

  checkDownloadPage(url, tlp, version)

  # display the test results as text
  puts ""
  puts "================="
  puts ""
  @tests.each { |t| t.map{|k, v| puts "#{k}: - #{v}"}}
  puts ""
  testentries(:W).each { |t| t.map{|k, v| puts "#{k}: - #{v}"}}
  testentries(:E).each { |t| t.map{|k, v| puts "#{k}: - #{v}"}}
  testentries(:F).each { |t| t.map{|k, v| puts "#{k}: - #{v}"}}
  puts ""
  if @fails > 0
    puts "NAK: #{url} had #{@fails} errors"
  else
    puts "OK: #{url} passed all the tests"
  end
  puts ""
end