| import collections | |
| import datetime | |
| import json | |
| import os | |
| import re | |
| import xml.sax.saxutils as saxutils | |
| import sys | |
| import traceback | |
| from optparse import OptionParser | |
| parser = OptionParser() | |
| parser.add_option("-v","--version",help="major version to filter on",dest="filterversion") | |
| parser.add_option("-e","--extratext",help="extra text to add to description",dest="extratext") | |
| parser.add_option("-i","--inputdirectory",help="directory of json files",dest="directory") | |
| parser.add_option("-d","--debug",help="enable debug output to stderr",dest="debug",action="store_true",default=False) | |
| parser.add_option("-l","--impactlink",help="URL for impact levels page",dest="impactlink",default="/security/impact_levels.html") | |
| parser.add_option("-n","--nopreamble",help="skip the page header/preamble output",dest="nopreamble",action="store_true",default=False) | |
| # To debug without getting flooded, put 1 working and 1 broken JSON file in tmp/ and pass --debug -i tmp | |
| (options,args) = parser.parse_args() | |
| re_fixedin = re.compile('(released in )?(?P<released>\\d\\.\\d\\.\\d+(-\\S+)?)( released)?'); | |
| def natural_sort_key(s, _nsre=re.compile('([0-9]+)')): | |
| return [int(text) if text.isdigit() else text.lower() | |
| for text in _nsre.split(s)] | |
| filterversion = options.filterversion or "" | |
| cves = [] | |
| entries = {} | |
| releasedates = {} # release -> [(date, cveid)] from each CVE's "x.y.z released" timeline entry | |
| DEFAULT_CVE_DATA_VERSION = "4.0" # Default (old) CVE data version | |
| # Sort order for CVEs within a release: most severe first. Anything else (e.g. "n/a") | |
| # sorts last. | |
| IMPACT_ORDER = {"critical": 0, "important": 1, "moderate": 2, "low": 3} | |
| # How many releases to list in the "Recent releases" line before the full index. | |
| RECENT_RELEASES = 5 | |
| def release_date(fixedv, dated): | |
| """The release date most of its CVEs agree on; warn about any that disagree. | |
| Returns None for -dev / never-released versions or when no date is known.""" | |
| if "-" in fixedv or not dated: | |
| return None | |
| counts = collections.Counter(d for d, _ in dated) | |
| best = max(sorted(counts), key=lambda d: counts[d]) | |
| odd = [cveid + "=" + d for d, cveid in dated if d != best] | |
| if odd: | |
| print("warning: release "+fixedv+" dated "+best+" by most CVEs, but "+", ".join(sorted(odd)), file=sys.stderr) | |
| return best | |
| def month_year(date): | |
| try: | |
| return datetime.date.fromisoformat(date).strftime("%b %Y") | |
| except ValueError: | |
| return date | |
| def plural_cves(n): | |
| return str(n)+" CVE"+("s" if n != 1 else "") | |
| for x in os.listdir(options.directory or "./"): | |
| if x.endswith(".json"): | |
| if options.debug: | |
| print(f"Processing: {x}", file=sys.stderr) | |
| try: | |
| fd = open(options.directory+x) | |
| cve = json.load(fd) | |
| cve["_filename"] = x | |
| cves.append(cve) | |
| except: | |
| print ("Ignoring due to error parsing: "+x) | |
| if options.debug: | |
| print(f"Skipped {x}: parse error", file=sys.stderr) | |
| continue | |
| # Filter on version and store by release(s) that fixed it | |
| for cve in cves: | |
| # Establish which version of CVE JSON we are dealing with | |
| data_version = cve.get("dataVersion", DEFAULT_CVE_DATA_VERSION) | |
| #print("%s: v%s" % (cve["_filename"], data_version), file=sys.stderr) | |
| if data_version == DEFAULT_CVE_DATA_VERSION: # Old style CVE | |
| cve["id"] = cve["CVE_data_meta"]["ID"] | |
| timearray = cve["timeline"] | |
| elif data_version.startswith("5"): # Newer style JSON | |
| cve["id"] = cve["cveMetadata"]["cveId"] | |
| timearray = cve["containers"]["cna"]["timeline"] | |
| else: | |
| print(f"unknown data version {data_version} in cve {cve['_filename']}", file=sys.stderr) | |
| sys.exit(1) | |
| if options.debug: | |
| print(f"Filtering: {cve['id']} from {cve['_filename']}", file=sys.stderr) | |
| matched = False | |
| for time in timearray: | |
| timed = time["value"] | |
| matcher = re_fixedin.match(timed); | |
| if (matcher and matcher.group('released').startswith(filterversion)): | |
| fixedin = matcher.group('released') | |
| if (not fixedin in entries): | |
| entries[fixedin] = [] | |
| releasedates[fixedin] = [] | |
| entries[fixedin].append(cve) | |
| releasedates[fixedin].append((time["time"].split('T')[0], cve["id"])) | |
| matched = True | |
| if options.debug and not matched: | |
| print(f"Skipped {cve['id']}: no matching version for filter '{filterversion}'", file=sys.stderr) | |
| # We want to sort on reverse number fixed, where our versions are dotted numbers, except some special cases | |
| # like never-fixed, or -dev fixed which should always appear first | |
| # Then we want to sort in the section by CVE name, handling numerics correctly | |
| lastfixedv = "" | |
| productname = "" | |
| sections = [] | |
| for k,v in sorted(entries.items(), key=lambda s: [int(u) if u.isdigit() else 999 for u in s[0].split(',')[0].split('.')], reverse=True): | |
| fixedv = k.split(",")[0] | |
| sectioncves = [] | |
| for cve in v: | |
| e = {} | |
| e['cveid'] = cve["id"] | |
| data_version = cve.get("dataVersion", DEFAULT_CVE_DATA_VERSION) | |
| if data_version.startswith("5"): | |
| try: | |
| e['impact'] = cve["containers"]["cna"]["metrics"][0]["other"]["content"]["text"] | |
| e['title'] = cve["containers"]["cna"]["title"] | |
| e['desc'] = cve["containers"]["cna"]["descriptions"][0]["value"] | |
| e['credit'] = [] | |
| if ("credits" in cve["containers"]["cna"]): | |
| for credit in cve["containers"]["cna"]["credits"]: | |
| e['credit'].append(credit["type"]+": "+credit["value"]) | |
| except Exception as err: | |
| sys.stderr.write("Missing data in " + cve["_filename"] + ": " + traceback.format_exc().replace('\n','\n ')) | |
| sys.exit(1); | |
| affects = [] | |
| product = cve["containers"]["cna"]["affected"][0] | |
| productname = product['product'] | |
| for ver in product["versions"]: | |
| base = "" | |
| appended = False | |
| if ver.get("version", "0") != "0": | |
| base = ver["version"] | |
| if "lessThanOrEqual" in ver: | |
| affects.append(f"{base} through {ver['lessThanOrEqual']}") | |
| appended = True | |
| if "lessThan" in ver: | |
| affects.append(f"{base} before {ver['lessThan']}") | |
| appended = True | |
| if not appended and base != "": | |
| affects.append(base) | |
| # Make a natural order sort | |
| affects.sort(reverse=True, key=natural_sort_key) | |
| e['affects'] = ", ".join(affects) | |
| e['timetable'] = []; | |
| for time in cve["containers"]["cna"].get("timeline", []): # Timeline may not be present in the JSON! | |
| timed = time["value"] | |
| if ("reported" in timed): | |
| timed = "Reported to security team" | |
| elif ("public" in timed): | |
| timed = "Issue public" | |
| elif ("release" in timed): | |
| timed = "Update "+timed | |
| e['timetable'].append([timed,time["time"].split('T')[0]]) | |
| sectioncves.append(e) | |
| else: | |
| e['impact'] = cve["impact"][0]["other"] | |
| e['title'] = cve["CVE_data_meta"]["TITLE"] | |
| e['desc'] = cve["description"]["description_data"][0]["value"] | |
| e['credit'] = [] | |
| if ("credit" in cve): | |
| for credit in cve["credit"]: | |
| e['credit'].append(credit["value"]) | |
| affects = [] | |
| product = cve["affects"]["vendor"]["vendor_data"][0]["product"]["product_data"][0] | |
| productname = product['product_name'] | |
| for ver in product["version"]["version_data"]: | |
| if (ver["version_affected"] == "="): | |
| affects.append(ver["version_value"]) | |
| elif (ver["version_affected"] == "?="): | |
| # We did ?= for "maybe affects" because no one checked | |
| affects.append(ver["version_value"]+"?") | |
| else: | |
| # Otherwise maybe we started doing things like "<2.7.8" | |
| affects.append(ver["version_affected"]+ver["version_value"]) | |
| # Make a natural order sort | |
| affects.sort(reverse=True, key=natural_sort_key) | |
| e['affects'] = ", ".join(affects) | |
| e['timetable'] = []; | |
| for time in cve.get("timeline", []): | |
| timed = time["value"] | |
| if ("reported" in timed): | |
| timed = "Reported to security team" | |
| elif ("public" in timed): | |
| timed = "Issue public" | |
| elif ("release" in timed): | |
| timed = "Update "+timed | |
| e['timetable'].append([timed,time["time"]]) | |
| sectioncves.append(e) | |
| # Within a release, sort by impact (most severe first), then by CVE id. | |
| sectioncves.sort(key=lambda e: (IMPACT_ORDER.get(e['impact'].lower(), 9), | |
| [int(u) if u.isdigit() else u for u in e['cveid'].split('-')])) | |
| sections.append({"cves":sectioncves,"fixed":fixedv,"product":productname, | |
| "date":release_date(fixedv, releasedates.get(k, []))}) | |
| # Everything is sorted and pretty, this should be some python template thing | |
| # We are generating html in markdown. Add the metadata first. | |
| if not options.nopreamble: | |
| print ("Title: "+productname+" "+filterversion+" vulnerabilities") | |
| print ("asf_headings: False") | |
| print ("") | |
| print ("<h1>"+productname+" "+filterversion+" vulnerabilities</h1>") | |
| print ("<p>This page lists all security vulnerabilities fixed in released versions of "+productname+" "+filterversion+". Each vulnerability is given a security <a href=\""+options.impactlink+"\">impact rating</a> by the Apache security team - please note that this rating may well vary from platform to platform. We also list the versions the flaw is known to affect, and where a flaw has not been verified list the version with a question mark.</p>") | |
| print ("<p>Please note that if a vulnerability is shown below as being fixed in a \"-dev\" release then this means that a fix has been applied to the development source tree and will be part of an upcoming full release.</p>") | |
| print ("<p>Please send comments or corrections for these vulnerabilities to the <a href=\"/security_report.html\">Security Team</a>.</p> <br/>") | |
| if (options.extratext): | |
| print ("<p>"+options.extratext+"</p><br/>") | |
| # The page covers many releases, so give readers a jump list: the most | |
| # recent few inline, the rest behind a collapsed <details>. | |
| def index_entry(s): | |
| n = len(s["cves"]) | |
| when = month_year(s["date"])+", " if s["date"] else "" | |
| return "<a href=\"#"+s["fixed"]+"\">"+s["fixed"]+"</a> ("+when+plural_cves(n)+")" | |
| print ("\n<nav class=\"cve-index\" aria-label=\"Releases on this page\">") | |
| print ("<p class=\"cve-recent\">Recent releases: " | |
| +" · ".join(index_entry(s) for s in sections[:RECENT_RELEASES])+"</p>") | |
| if len(sections) > RECENT_RELEASES: | |
| print ("<details><summary>All "+str(len(sections))+" releases</summary>") | |
| print ("<ul>") | |
| for s in sections: | |
| print ("<li>"+index_entry(s)+"</li>") | |
| print ("</ul>") | |
| print ("</details>") | |
| print ("</nav>\n") | |
| for sectioncves in sections: | |
| count = len(sectioncves["cves"]) | |
| sevcount = {} | |
| for e in sectioncves["cves"]: | |
| sevcount[e['impact']] = sevcount.get(e['impact'], 0) + 1 | |
| sevparts = ["<strong>"+str(sevcount[k])+"</strong> "+k for k in ("critical", "important", "moderate", "low") if sevcount.get(k)] | |
| sevparts += ["<strong>"+str(sevcount[k])+"</strong> "+k for k in sorted(sevcount) if k not in ("critical", "important", "moderate", "low")] | |
| summary = "" | |
| if sectioncves["date"]: | |
| summary += "Released <time datetime=\""+sectioncves["date"]+"\">"+sectioncves["date"]+"</time> · " | |
| summary += "<strong>"+plural_cves(count)+"</strong>" | |
| if sevparts: | |
| summary += ": "+", ".join(sevparts) | |
| print ("\n<h1 id=\""+sectioncves["fixed"]+"\">Fixed in "+sectioncves["product"]+" "+sectioncves["fixed"]+"</h1>") | |
| print ("<p class=\"cve-summary\">"+summary+"</p>") | |
| print ("<dl class=\"cve-list\">\n") | |
| for e in sectioncves["cves"]: | |
| html = "<dt><h3 id=\""+e['cveid']+"\"><a href=\"https://www.cve.org/CVERecord?id="+e['cveid']+"\">"+e['cveid']+"</a> ("+e['impact']+"): <name name=\""+e['cveid']+"\">"+saxutils.escape(e['title'])+"</name>\n"; | |
| html += "<a class=\"headerlink\" href=\"#"+e['cveid']+"\" title=\"Permanent link\">¶</a></h3></dt>\n"; | |
| desc = saxutils.escape(e['desc']) | |
| desc = re.sub(r'\n','</p><p>', desc) | |
| html += "<dd><p>"+desc+"</p>\n" | |
| if (e['credit']): | |
| html += "<p>Acknowledgements:" | |
| if len(e['credit']) == 1: | |
| html += " " + saxutils.escape(e['credit'][0]) + "</p>\n" | |
| else: | |
| html += "</p>\n<ul>\n" | |
| for credit in e['credit']: | |
| html += "<li>" + saxutils.escape(credit) + "</li>\n" | |
| html += "</ul>\n" | |
| html += "<table class=\"cve\">" | |
| e['timetable'].append(["Affects",e['affects']]); | |
| for ti in e['timetable']: | |
| html+= "<tr><td class=\"cve-header\">"+ti[0]+"</td><td class=\"cve-value\">"+ti[1]+"</td></tr>\n" | |
| html+= "</table></dd>" | |
| print (html) | |
| print ("</dl></br/>") | |
| print ("<p class=\"cve-backtop\"><a href=\"#apcontents\">↑ Back to top</a></p>") | |