blob: c59e9a087b26e536cf5d9a22742585f061b5caf7 [file]
#**************************************************************
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
#**************************************************************
# usage: pkgcheck.py <outdir> <pkg.json>
# ODF 1.2 Part 3 2.2.1, 3.2, 3.3 and 3.4.1 on every package in <outdir>. Each finding is
# "shall: [rule] ..." or "should: [rule] ...". Schema validity of the manifest is validate.py's job.
import sys, os, re, zipfile, json, struct, subprocess, shutil, warnings
from xml.etree import ElementTree as ET
D = os.path.dirname(os.path.abspath(__file__))
JING = ["java", "-jar", os.environ.get("ODFV_JING", "/Applications/XMLauthor/lib/jing.jar"), "-i"]
DSIG_SCHEMA = D + "/schema/OpenDocument-v1.2-os-dsig-schema.rng"
MANI = "{urn:oasis:names:tc:opendocument:xmlns:manifest:1.0}"
DSIG = "{urn:oasis:names:tc:opendocument:xmlns:digitalsignature:1.0}"
ERR = re.compile(r"^(.*?):(\d+):(\d+): (error|fatal|warning): (.*)$")
def check_package(path, work):
"""Returns [(entry, level, rule, message)] for one package."""
out = []
def add(entry, level, rule, msg):
out.append((entry, level, rule, msg))
z = zipfile.ZipFile(path)
infos = z.infolist()
names = [i.filename for i in infos]
for n in sorted({n for n in names if names.count(n) > 1}):
add(n, "shall", "zip-duplicate", "zip entry name occurs more than once")
for i in infos:
if i.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED):
add(i.filename, "shall", "zip-method", "compression method %d is neither STORED nor DEFLATED (2.2.1 A)" % i.compress_type)
if i.filename.endswith("/"):
add(i.filename, "should", "zip-dir-entry", "directories have no zip entries (4.3)")
files = {n for n in names if not n.endswith("/")}
# mimetype (3.3)
mime = None
if "mimetype" not in files:
add("mimetype", "should", "mimetype-missing", "no mimetype file (2.2.1 C)")
else:
mi = z.getinfo("mimetype")
first = min(infos, key=lambda i: i.header_offset)
if first.filename != "mimetype" or mi.header_offset != 0:
add("mimetype", "shall", "mimetype-first", "mimetype is not the first file of the zip (3.3)")
with open(path, "rb") as f:
f.seek(mi.header_offset)
hdr = f.read(30)
if len(hdr) == 30 and hdr[:4] == b"PK\x03\x04":
method, extralen = struct.unpack("<H", hdr[8:10])[0], struct.unpack("<H", hdr[28:30])[0]
if method != zipfile.ZIP_STORED:
add("mimetype", "shall", "mimetype-compressed", "mimetype is compressed, local header method %d (3.3)" % method)
if extralen:
add("mimetype", "shall", "mimetype-extra", "mimetype has a %d byte extra field in its local header (3.3)" % extralen)
else:
add("mimetype", "shall", "mimetype-first", "no local file header at the recorded offset")
if mi.compress_type != zipfile.ZIP_STORED:
add("mimetype", "shall", "mimetype-compressed", "mimetype is compressed, central directory method %d (3.3)" % mi.compress_type)
raw = z.read("mimetype")
try:
mime = raw.decode("ascii")
except UnicodeDecodeError:
add("mimetype", "shall", "mimetype-ascii", "mimetype content is not ASCII (3.3)")
# META-INF contents (2.2.1 D, E)
for n in sorted(files):
if n.startswith("META-INF/") and n != "META-INF/manifest.xml" and "signatures" not in n.rsplit("/", 1)[-1]:
add(n, "shall", "meta-inf-file", "file other than manifest.xml and signature files in META-INF (2.2.1 E)")
# manifest (2.2.1 B, 3.2)
entries = None
if "META-INF/manifest.xml" not in files:
add("META-INF/manifest.xml", "shall", "manifest-missing", "no META-INF/manifest.xml (2.2.1 B)")
else:
try:
root = ET.fromstring(z.read("META-INF/manifest.xml"))
if root.tag != MANI + "manifest":
add("META-INF/manifest.xml", "shall", "manifest-root", "root element is not manifest:manifest (2.2.1 B.2)")
else:
entries = [e for e in root if e.tag == MANI + "file-entry"]
except ET.ParseError as e:
add("META-INF/manifest.xml", "shall", "manifest-malformed", "not well-formed: %s (2.2.1 B.1)" % e)
if entries is not None:
paths = [e.get(MANI + "full-path") for e in entries]
for p in sorted({p for p in paths if paths.count(p) > 1}, key=str):
add("META-INF/manifest.xml", "shall", "manifest-duplicate", "more than one file-entry for %s (3.2)" % p)
for p in paths:
if p in ("META-INF/manifest.xml", "mimetype"):
add("META-INF/manifest.xml", "shall", "manifest-self", "file-entry for %s (3.2)" % p)
elif p and p.endswith("/") and p != "/":
if p + "content.xml" not in files and p + "styles.xml" not in files:
add("META-INF/manifest.xml", "should", "manifest-dir-entry", "file-entry for %s, a directory without a document (4.3)" % p)
elif p not in files and p != "/":
add("META-INF/manifest.xml", "should", "manifest-dangling", "file-entry for %s, which is not in the package (3.2)" % p)
for n in sorted(files):
if n != "mimetype" and not n.startswith("META-INF/") and n not in paths:
add(n, "shall", "manifest-coverage", "no file-entry in the manifest (3.2)")
root_entry = [e for e in entries if e.get(MANI + "full-path") == "/"]
if "mimetype" in files and not root_entry:
add("META-INF/manifest.xml", "shall", "manifest-root-entry", 'no file-entry for "/" although the package has a mimetype file (3.2)')
for e in root_entry:
mt = e.get(MANI + "media-type")
if "mimetype" not in files:
add("mimetype", "shall", "mimetype-missing", 'file-entry for "/" but no mimetype file (3.3)')
elif mime is not None and mt != mime:
add("mimetype", "shall", "mimetype-mismatch", 'mimetype is "%s" but the "/" media-type is "%s" (3.3)' % (mime, mt))
# encrypted entries (3.4.1)
for e in entries:
if e.find(MANI + "encryption-data") is None:
continue
p = e.get(MANI + "full-path")
size = e.get(MANI + "size")
if size is None or not re.fullmatch(r"[0-9]+", size):
add(p, "shall", "encrypted-size", "encrypted entry without a valid manifest:size (3.4.1, 4.8.13)")
if p in files and z.getinfo(p).compress_type != zipfile.ZIP_STORED:
add(p, "shall", "encrypted-method", "encrypted entry is not flagged STORED in the zip (3.4.1)")
# signature files (2.2.1 D)
sigs = sorted(n for n in files if n.startswith("META-INF/") and "signatures" in n.rsplit("/", 1)[-1])
for n in sigs:
try:
root = ET.fromstring(z.read(n))
except ET.ParseError as e:
add(n, "shall", "signature-malformed", "not well-formed: %s (2.2.1 D.1)" % e)
continue
if root.tag != DSIG + "document-signatures":
add(n, "shall", "signature-root", "root element is not dsig:document-signatures (2.2.1 D.2)")
continue
p = os.path.join(work, os.path.basename(path), n)
os.makedirs(os.path.dirname(p), exist_ok=True)
open(p, "wb").write(z.read(n))
r = subprocess.run(JING + [DSIG_SCHEMA, p], capture_output=True, text=True)
for line in (r.stdout + r.stderr).splitlines():
m = ERR.match(line)
if m:
add(n, "shall", "signature-schema", "%s:%s %s (2.2.1 D.3)" % (m.group(2), m.group(3), m.group(5)))
elif line.strip():
add(n, "shall", "signature-schema", line.strip())
return out
def main():
outdir, pkgjson = sys.argv[1], sys.argv[2]
if not os.path.abspath(pkgjson).startswith(os.path.join(D, "runs") + os.sep):
sys.exit("pkgcheck.py: <pkg.json> must live under " + os.path.join(D, "runs"))
work = os.path.abspath(pkgjson[:-5] + ".xml")
shutil.rmtree(work, ignore_errors=True)
errors, bad_zip = [], []
pkgs = sorted(p for p in os.listdir(outdir) if not p.startswith("."))
for pkg in pkgs:
try:
found = check_package(os.path.join(outdir, pkg), work)
except zipfile.BadZipFile:
bad_zip.append(pkg); continue
for entry, level, rule, msg in found:
errors.append([pkg + "/" + entry, 0, 0, "%s: [%s] %s" % (level, rule, msg)])
json.dump({"packages": pkgs, "bad_zip": bad_zip, "errors": errors}, open(pkgjson, "w"))
shall = [e for e in errors if e[3].startswith("shall:")]
print(f"package check: {len(shall)} shall and {len(errors) - len(shall)} should findings in "
f"{len({e[0].split('/')[0] for e in errors})} of {len(pkgs)} packages, bad zips: {bad_zip}")
rules = {}
for e in errors:
k = e[3].split("]")[0] + "]"
rules[k] = rules.get(k, 0) + 1
for k in sorted(rules):
print(f" {rules[k]:6d} {k}")
main()