dev-tools/scripts/buildAndPushRelease.py - lucene - Git at Google

 #!/usr/bin/env python3
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
 # The ASF licenses this file to You under the Apache License, Version 2.0
 # (the "License"); you may not use this file except in compliance with
 # the License.  You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.

 import argparse
 import datetime
 import getpass
 import os
 import re
 import subprocess
 import sys
 import textwrap
 import time
 import urllib.request
 import xml.etree.ElementTree as ET
 from pathlib import Path
 from subprocess import TimeoutExpired

 import scriptutil

 LOG = "/tmp/release.log"
 dev_mode = False


 def log(msg: str):
   f = open(LOG, mode="ab")
   f.write(msg.encode("utf-8"))
   f.close()


 def run(command: str):
   log("\n\n%s: RUN: %s\n" % (datetime.datetime.now(), command))
   if os.system("%s >> %s 2>&1" % (command, LOG)):
     msg = "    FAILED: %s [see log %s]" % (command, LOG)
     print(msg)
     raise RuntimeError(msg)


 def runAndSendGPGPassword(command: str, password: str):
   p = subprocess.Popen(command, shell=True, bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE)
   f = open(LOG, "ab")
   while True:
     assert p.stdout
     assert p.stdin
     p.stdout.flush()
     line = p.stdout.readline()
     if len(line) == 0:
       break
     f.write(line)
     if line.find(b"Enter GPG keystore password:") != -1:
       time.sleep(1.0)
       p.stdin.write((password + "\n").encode("UTF-8"))
       p.stdin.write(b"\n")

   try:
     result = p.wait(timeout=120)
     if result != 0:
       msg = "    FAILED: %s [see log %s]" % (command, LOG)
       print(msg)
       raise RuntimeError(msg)
   except TimeoutExpired:
     msg = "    FAILED: %s [timed out after 2 minutes; see log %s]" % (command, LOG)
     print(msg)
     raise RuntimeError(msg)


 def load(urlString: str, encoding: str = "utf-8"):
   try:
     content = urllib.request.urlopen(urlString).read().decode(encoding)
   except Exception as e:
     print("Retrying download of url %s after exception: %s" % (urlString, e))
     content = urllib.request.urlopen(urlString).read().decode(encoding)
   return content


 def getGitRev():
   if not dev_mode:
     status = os.popen("git status").read().strip()
     if "nothing to commit, working directory clean" not in status and "nothing to commit, working tree clean" not in status:
       raise RuntimeError("git clone is dirty:\n\n%s" % status)
     if "Your branch is ahead of" in status:
       raise RuntimeError("Your local branch is ahead of the remote? git status says:\n%s" % status)
     print("  git clone is clean")
   else:
     print("  Ignoring dirty git clone due to dev-mode")
   return os.popen("git rev-parse HEAD").read().strip()


 def prepare(root: str, version: str, pause_before_sign: bool, gpg_key_id: str | None, gpg_password: str | None, gpg_home: str | None = None, sign_gradle: bool = False):
   print()
   print("Prepare release...")
   if os.path.exists(LOG):
     os.remove(LOG)

   if not dev_mode:
     os.chdir(root)
     print("  git pull...")
     run("git pull")
   else:
     print("  Development mode, not running git pull")

   rev = getGitRev()
   print("  git rev: %s" % rev)
   log("\nGIT rev: %s\n" % rev)

   print("  Check DOAP files")
   checkDOAPfiles(version)

   if not dev_mode:
     print("  ./gradlew --stacktrace --no-daemon clean")
     run("./gradlew --stacktrace --no-daemon clean")
     print("  ./gradlew --stacktrace --no-daemon check")
     run("./gradlew --stacktrace --no-daemon check")
   else:
     print("  skipping precommit check due to dev-mode")

   if pause_before_sign:
     input("Tests complete! Please press ENTER to proceed to assembleRelease: ")

   print("  prepare-release")
   cmd = "./gradlew --stacktrace --no-daemon assembleRelease -Dversion.release=%s" % version
   if gpg_key_id is not None:
     cmd += " -Psign --max-workers 2"
     if sign_gradle:
       print("  Signing method is gradle java-plugin")
       cmd += ' -Psigning.keyId="%s"' % gpg_key_id
       if gpg_home is not None:
         cmd += ' -Psigning.secretKeyRingFile="%s"' % os.path.join(gpg_home, "secring.gpg")
       if gpg_password is not None:
         # Pass gpg passphrase as env.var to gradle rather than as plaintext argument
         os.environ["ORG_GRADLE_PROJECT_signingPassword"] = gpg_password
     else:
       print("  Signing method is gpg tool")
       cmd += ' -PuseGpg=true -Psigning.gnupg.keyName="%s"' % gpg_key_id
       if gpg_home is not None:
         cmd += ' -Psigning.gnupg.homeDir="%s"' % gpg_home

   print("  Running: %s" % cmd)
   if gpg_password is not None:
     runAndSendGPGPassword(cmd, gpg_password)
   else:
     run(cmd)

   print("  done!")
   print()
   return rev


 reVersion1 = re.compile(r"\>(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?/\<", re.IGNORECASE)
 reVersion2 = re.compile(r"-(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?\.zip<", re.IGNORECASE)
 reDoapRevision = re.compile(r"(\d+)\.(\d+)(?:\.(\d+))?(-alpha|-beta)?", re.IGNORECASE)


 def checkDOAPfiles(version: str):
   # In Lucene DOAP file, verify presence of all releases less than the one being produced.
   errorMessages: list[str] = []
   for product in ["lucene"]:
     url = "https://archive.apache.org/dist/lucene/%s" % ("java" if product == "lucene" else product)
     distpage = load(url)
     releases: set[str] = set()
     for regex in reVersion1, reVersion2:
       for tup in regex.findall(distpage):
         if tup[0] in ("1", "2"):  # Ignore 1.X and 2.X releases
           continue
         releases.add(normalizeVersion(tup))
     doapNS = "{http://usefulinc.com/ns/doap#}"
     xpathRevision = f"{doapNS}Project/{doapNS}release/{doapNS}Version/{doapNS}revision"
     doapFile = "dev-tools/doap/%s.rdf" % product
     treeRoot = ET.parse(doapFile).getroot()
     doapRevisions: set[str] = set()
     for revision in treeRoot.findall(xpathRevision):
       if revision.text and (match := reDoapRevision.match(revision.text)):
         if match.group(1) not in ("0", "1", "2"):  # Ignore 0.X, 1.X and 2.X revisions
           doapRevisions.add(normalizeVersion(match.groups()))
       else:
         errorMessages.append("ERROR: Failed to parse revision: %s in %s" % (revision.text, doapFile))
     missingDoapRevisions: set[str] = set()
     for release in releases:
       if release not in doapRevisions and release < version:  # Ignore releases greater than the one being produced
         missingDoapRevisions.add(release)
     if len(missingDoapRevisions) > 0:
       errorMessages.append("ERROR: Missing revision(s) in %s: %s" % (doapFile, ", ".join(sorted(missingDoapRevisions))))
   if len(errorMessages) > 0:
     raise RuntimeError("\n%s\n(Hint: copy/paste from the stable branch version of the file(s).)" % "\n".join(errorMessages))


 def normalizeVersion(tup: tuple[str, ...]):
   suffix = ""
   if tup[-1] is not None and tup[-1].lower() == "-alpha":
     tup = tup[: (len(tup) - 1)]
     suffix = "-ALPHA"
   elif tup[-1] is not None and tup[-1].lower() == "-beta":
     tup = tup[: (len(tup) - 1)]
     suffix = "-BETA"
   while tup[-1] in ("", None):
     tup = tup[: (len(tup) - 1)]
   while len(tup) < 3:
     tup = tup + ("0",)
   return ".".join(tup) + suffix


 def pushLocal(version: str, root: str, rcNum: int, localDir: str):
   print("Push local [%s]..." % localDir)
   Path(localDir).mkdir(parents=True)

   lucene_dist_dir = "%s/lucene/distribution/build/release" % root
   rev = open("%s/lucene/distribution/build/release/.gitrev" % root, encoding="UTF-8").read()

   dir = "lucene-%s-RC%d-rev-%s" % (version, rcNum, rev)
   Path("%s/%s/lucene" % (localDir, dir)).mkdir(parents=True)
   print("  Lucene")
   os.chdir(lucene_dist_dir)
   print("    archive...")
   if os.path.exists("lucene.tar"):
     os.remove("lucene.tar")
   run("tar cf lucene.tar *")

   os.chdir("%s/%s/lucene" % (localDir, dir))
   print("    extract...")
   run('tar xf "%s/lucene.tar"' % lucene_dist_dir)
   os.remove("%s/lucene.tar" % lucene_dist_dir)
   os.chdir("..")

   print("  chmod...")
   run("chmod -R a+rX-w .")

   print("  done!")
   return "file://%s/%s" % (Path(localDir).resolve(), dir)


 def read_version(_path: str):
   return scriptutil.find_current_version()


 def parse_config():
   epilogue = textwrap.dedent("""
     Example usage for a Release Manager:
     python3 -u dev-tools/scripts/buildAndPushRelease.py --push-local /tmp/releases/6.0.1 --sign 6E68DA61 --rc-num 1
   """)
   description = "Utility to build, push, and test a release."
   parser = argparse.ArgumentParser(description=description, epilog=epilogue, formatter_class=argparse.RawDescriptionHelpFormatter)
   parser.add_argument("--no-prepare", dest="prepare", default=True, action="store_false", help="Use the already built release in the provided checkout")
   parser.add_argument("--local-keys", metavar="PATH", help="Uses local KEYS file to validate presence of RM's gpg key")
   parser.add_argument("--push-local", metavar="PATH", help="Push the release to the local path")
   parser.add_argument("--pause-before-sign", default=False, action="store_true", help="Pause for user confirmation before the assembleRelease step (to prevent timeout on gpg pinentry")
   parser.add_argument("--sign", metavar="KEYID", help="Sign the release with the given gpg key")
   parser.add_argument(
     "--sign-method-gradle",
     dest="sign_method_gradle",
     default=False,
     action="store_true",
     help="Use Gradle built-in GPG signing instead of gpg command for signing artifacts.  This may require --gpg-secring argument if your keychain cannot be resolved automatically.",
   )
   parser.add_argument(
     "--gpg-pass-noprompt",
     dest="gpg_pass_noprompt",
     default=False,
     action="store_true",
     help="""
     Do not prompt for gpg passphrase. For the default gnupg method, this means your gpg-agent
     needs a non-TTY pin-entry program. For gradle signing method, passphrase must be provided
     in gradle.properties or by env.var/sysprop. See ./gradlew helpPublishing for more info
     """.strip(),
   )
   parser.add_argument("--gpg-home", metavar="PATH", help="Path to gpg home containing your secring.gpg Optional, will use $HOME/.gnupg/secring.gpg by default")
   parser.add_argument("--rc-num", metavar="NUM", type=int, default=1, help="Release Candidate number.  Default: 1")
   parser.add_argument("--root", metavar="PATH", default=".", help='Root of Git working tree for lucene.  Default: "." (the current directory)')
   parser.add_argument("--logfile", metavar="PATH", help="Specify log file path (default /tmp/release.log)")
   parser.add_argument("--dev-mode", default=False, action="store_true", help="Enable development mode, which disables some strict checks")
   config = parser.parse_args()

   if not config.prepare and config.sign:
     parser.error("Cannot sign already built release")
   if config.push_local is not None and os.path.exists(config.push_local):
     parser.error("Cannot push to local path that already exists")
   if config.rc_num <= 0:
     parser.error("Release Candidate number must be a positive integer")
   if not os.path.isdir(config.root):
     parser.error('Root path "%s" is not a directory' % config.root)
   if config.local_keys is not None and not os.path.exists(config.local_keys):
     parser.error('Local KEYS file "%s" not found' % config.local_keys)
   if config.gpg_home and not os.path.exists(os.path.join(config.gpg_home, "secring.gpg")):
     parser.error("Specified gpg home %s does not exist or does not contain a secring.gpg" % config.gpg_home)
   global dev_mode
   if config.dev_mode:
     print("Enabling development mode - DO NOT USE FOR ACTUAL RELEASE!")
     dev_mode = True
   cwd = os.getcwd()
   os.chdir(config.root)
   config.root = os.getcwd()  # Absolutize root dir
   if os.system("git rev-parse") or len([d for d in ("dev-tools", "lucene") if os.path.isdir(d)]) != 2:
     parser.error('Root path "%s" is not a valid lucene checkout' % config.root)
   os.chdir(cwd)
   global LOG
   if config.logfile:
     LOG = config.logfile
   print("Logfile is: %s" % LOG)

   config.version = read_version(config.root)
   print("Building version: %s" % config.version)

   return config


 def check_cmdline_tools():  # Fail fast if there are cmdline tool problems
   if os.system("git --version >/dev/null 2>/dev/null"):
     raise RuntimeError('"git --version" returned a non-zero exit code.')


 def check_key_in_keys(gpgKeyID: str | None, local_keys: str | None):
   if gpgKeyID is not None:
     print("  Verify your gpg key is in the main KEYS file")
     if local_keys is not None:
       print("    Using local KEYS file %s" % local_keys)
       keysFileText = open(local_keys, encoding="iso-8859-1").read()
       keysFileLocation = local_keys
     else:
       keysFileURL = "https://archive.apache.org/dist/lucene/KEYS"
       keysFileLocation = keysFileURL
       print("    Using online KEYS file %s" % keysFileURL)
       keysFileText = load(keysFileURL, encoding="iso-8859-1")
     if len(gpgKeyID) > 2 and gpgKeyID[0:2] == "0x":
       gpgKeyID = gpgKeyID[2:]
     if len(gpgKeyID) > 40:
       gpgKeyID = gpgKeyID.replace(" ", "")
     if len(gpgKeyID) == 8:
       gpgKeyID8Char = "%s %s" % (gpgKeyID[0:4], gpgKeyID[4:8])
       re_to_match = r"^pub .*\n\s+(\w{4} \w{4} \w{4} \w{4} \w{4}  \w{4} \w{4} \w{4} %s|\w{32}%s)" % (gpgKeyID8Char, gpgKeyID)
     elif len(gpgKeyID) == 40:
       gpgKeyID40Char = "%s %s %s %s %s  %s %s %s %s %s" % (
         gpgKeyID[0:4],
         gpgKeyID[4:8],
         gpgKeyID[8:12],
         gpgKeyID[12:16],
         gpgKeyID[16:20],
         gpgKeyID[20:24],
         gpgKeyID[24:28],
         gpgKeyID[28:32],
         gpgKeyID[32:36],
         gpgKeyID[36:],
       )
       re_to_match = r"^pub .*\n\s+(%s|%s)" % (gpgKeyID40Char, gpgKeyID)
     else:
       print("Invalid gpg key id format [%s]. Must be 8 byte short ID or 40 byte fingerprint, with or without 0x prefix, no spaces." % gpgKeyID)
       exit(2)
     if re.search(re_to_match, keysFileText, re.MULTILINE):
       print("    Found key %s in KEYS file at %s" % (gpgKeyID, keysFileLocation))
     else:
       print("    ERROR: Did not find your key %s in KEYS file at %s. Please add it and try again." % (gpgKeyID, keysFileLocation))
       if local_keys is not None:
         print("           You are using a local KEYS file. Make sure it is up to date or validate against the online version")
       exit(2)


 def resolve_gpghome():
   for p in [
     # Linux, macos
     os.path.join(os.path.expanduser("~"), ".gnupg"),
     # Windows 10
     os.path.expandvars(r"%APPDATA%\GnuPG"),
     # TODO: Should we support Cygwin?
   ]:
     if os.path.exists(os.path.join(p, "secring.gpg")):
       return p
   return None


 def main():
   check_cmdline_tools()

   c = parse_config()
   gpg_home = None

   if c.sign:
     sys.stdout.flush()
     c.key_id = c.sign
     check_key_in_keys(c.key_id, c.local_keys)
     if c.gpg_home is not None:
       print("Using custom gpg-home: %s" % c.gpg_home)
       gpg_home = c.gpg_home
     if c.sign_method_gradle:
       if gpg_home is None:
         resolved_gpg_home = resolve_gpghome()
         if resolved_gpg_home is not None:
           print("Resolved gpg home to %s" % resolved_gpg_home)
           gpg_home = resolved_gpg_home
         else:
           print("WARN: Could not locate your gpg secret keyring, and --gpg-home not specified.")
           print("      Falling back to location configured in gradle.properties.")
           print("      See 'gradlew helpPublishing' for details.")
           gpg_home = None
     if c.gpg_pass_noprompt:
       print("Will not prompt for gpg password. Make sure your signing setup supports this.")
       c.key_password = None
     else:
       c.key_password = getpass.getpass("Enter GPG keystore password: ")
   else:
     c.key_id = None
     c.key_password = None

   if c.prepare:
     prepare(c.root, c.version, c.pause_before_sign, c.key_id, c.key_password, gpg_home=gpg_home, sign_gradle=c.sign_method_gradle)
   else:
     os.chdir(c.root)

   if c.push_local:
     url = pushLocal(c.version, c.root, c.rc_num, c.push_local)
   else:
     url = None

   if url is not None:
     print("  URL: %s" % url)
     print("Next run the smoker tester:")
     p = re.compile(".*/")
     m = p.match(sys.argv[0])
     assert m
     if not c.sign:
       signed = "--not-signed"
     else:
       signed = ""
     print("%s -u %ssmokeTestRelease.py %s %s" % (sys.executable, m.group(), signed, url))


 if __name__ == "__main__":
   try:
     main()
   except KeyboardInterrupt:
     print("Keyboard interrupt...exiting")
	#!/usr/bin/env python3
	# Licensed to the Apache Software Foundation (ASF) under one or more
	# contributor license agreements. See the NOTICE file distributed with
	# this work for additional information regarding copyright ownership.
	# The ASF licenses this file to You under the Apache License, Version 2.0
	# (the "License"); you may not use this file except in compliance with
	# the License. You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.

	import argparse
	import datetime
	import getpass
	import os
	import re
	import subprocess
	import sys
	import textwrap
	import time
	import urllib.request
	import xml.etree.ElementTree as ET
	from pathlib import Path
	from subprocess import TimeoutExpired

	import scriptutil

	LOG = "/tmp/release.log"
	dev_mode = False


	def log(msg: str):
	f = open(LOG, mode="ab")
	f.write(msg.encode("utf-8"))
	f.close()


	def run(command: str):
	log("\n\n%s: RUN: %s\n" % (datetime.datetime.now(), command))
	if os.system("%s >> %s 2>&1" % (command, LOG)):
	msg = " FAILED: %s [see log %s]" % (command, LOG)
	print(msg)
	raise RuntimeError(msg)


	def runAndSendGPGPassword(command: str, password: str):
	p = subprocess.Popen(command, shell=True, bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE)
	f = open(LOG, "ab")
	while True:
	assert p.stdout
	assert p.stdin
	p.stdout.flush()
	line = p.stdout.readline()
	if len(line) == 0:
	break
	f.write(line)
	if line.find(b"Enter GPG keystore password:") != -1:
	time.sleep(1.0)
	p.stdin.write((password + "\n").encode("UTF-8"))
	p.stdin.write(b"\n")

	try:
	result = p.wait(timeout=120)
	if result != 0:
	msg = " FAILED: %s [see log %s]" % (command, LOG)
	print(msg)
	raise RuntimeError(msg)
	except TimeoutExpired:
	msg = " FAILED: %s [timed out after 2 minutes; see log %s]" % (command, LOG)
	print(msg)
	raise RuntimeError(msg)


	def load(urlString: str, encoding: str = "utf-8"):
	try:
	content = urllib.request.urlopen(urlString).read().decode(encoding)
	except Exception as e:
	print("Retrying download of url %s after exception: %s" % (urlString, e))
	content = urllib.request.urlopen(urlString).read().decode(encoding)
	return content


	def getGitRev():
	if not dev_mode:
	status = os.popen("git status").read().strip()
	if "nothing to commit, working directory clean" not in status and "nothing to commit, working tree clean" not in status:
	raise RuntimeError("git clone is dirty:\n\n%s" % status)
	if "Your branch is ahead of" in status:
	raise RuntimeError("Your local branch is ahead of the remote? git status says:\n%s" % status)
	print(" git clone is clean")
	else:
	print(" Ignoring dirty git clone due to dev-mode")
	return os.popen("git rev-parse HEAD").read().strip()


	def prepare(root: str, version: str, pause_before_sign: bool, gpg_key_id: str \| None, gpg_password: str \| None, gpg_home: str \| None = None, sign_gradle: bool = False):
	print()
	print("Prepare release...")
	if os.path.exists(LOG):
	os.remove(LOG)

	if not dev_mode:
	os.chdir(root)
	print(" git pull...")
	run("git pull")
	else:
	print(" Development mode, not running git pull")

	rev = getGitRev()
	print(" git rev: %s" % rev)
	log("\nGIT rev: %s\n" % rev)

	print(" Check DOAP files")
	checkDOAPfiles(version)

	if not dev_mode:
	print(" ./gradlew --stacktrace --no-daemon clean")
	run("./gradlew --stacktrace --no-daemon clean")
	print(" ./gradlew --stacktrace --no-daemon check")
	run("./gradlew --stacktrace --no-daemon check")
	else:
	print(" skipping precommit check due to dev-mode")

	if pause_before_sign:
	input("Tests complete! Please press ENTER to proceed to assembleRelease: ")

	print(" prepare-release")
	cmd = "./gradlew --stacktrace --no-daemon assembleRelease -Dversion.release=%s" % version
	if gpg_key_id is not None:
	cmd += " -Psign --max-workers 2"
	if sign_gradle:
	print(" Signing method is gradle java-plugin")
	cmd += ' -Psigning.keyId="%s"' % gpg_key_id
	if gpg_home is not None:
	cmd += ' -Psigning.secretKeyRingFile="%s"' % os.path.join(gpg_home, "secring.gpg")
	if gpg_password is not None:
	# Pass gpg passphrase as env.var to gradle rather than as plaintext argument
	os.environ["ORG_GRADLE_PROJECT_signingPassword"] = gpg_password
	else:
	print(" Signing method is gpg tool")
	cmd += ' -PuseGpg=true -Psigning.gnupg.keyName="%s"' % gpg_key_id
	if gpg_home is not None:
	cmd += ' -Psigning.gnupg.homeDir="%s"' % gpg_home

	print(" Running: %s" % cmd)
	if gpg_password is not None:
	runAndSendGPGPassword(cmd, gpg_password)
	else:
	run(cmd)

	print(" done!")
	print()
	return rev


	reVersion1 = re.compile(r"\>(\d+)\.(\d+)\.(\d+)(-alpha\|-beta)?/\<", re.IGNORECASE)
	reVersion2 = re.compile(r"-(\d+)\.(\d+)\.(\d+)(-alpha\|-beta)?\.zip<", re.IGNORECASE)
	reDoapRevision = re.compile(r"(\d+)\.(\d+)(?:\.(\d+))?(-alpha\|-beta)?", re.IGNORECASE)


	def checkDOAPfiles(version: str):
	# In Lucene DOAP file, verify presence of all releases less than the one being produced.
	errorMessages: list[str] = []
	for product in ["lucene"]:
	url = "https://archive.apache.org/dist/lucene/%s" % ("java" if product == "lucene" else product)
	distpage = load(url)
	releases: set[str] = set()
	for regex in reVersion1, reVersion2:
	for tup in regex.findall(distpage):
	if tup[0] in ("1", "2"): # Ignore 1.X and 2.X releases
	continue
	releases.add(normalizeVersion(tup))
	doapNS = "{http://usefulinc.com/ns/doap#}"
	xpathRevision = f"{doapNS}Project/{doapNS}release/{doapNS}Version/{doapNS}revision"
	doapFile = "dev-tools/doap/%s.rdf" % product
	treeRoot = ET.parse(doapFile).getroot()
	doapRevisions: set[str] = set()
	for revision in treeRoot.findall(xpathRevision):
	if revision.text and (match := reDoapRevision.match(revision.text)):
	if match.group(1) not in ("0", "1", "2"): # Ignore 0.X, 1.X and 2.X revisions
	doapRevisions.add(normalizeVersion(match.groups()))
	else:
	errorMessages.append("ERROR: Failed to parse revision: %s in %s" % (revision.text, doapFile))
	missingDoapRevisions: set[str] = set()
	for release in releases:
	if release not in doapRevisions and release < version: # Ignore releases greater than the one being produced
	missingDoapRevisions.add(release)
	if len(missingDoapRevisions) > 0:
	errorMessages.append("ERROR: Missing revision(s) in %s: %s" % (doapFile, ", ".join(sorted(missingDoapRevisions))))
	if len(errorMessages) > 0:
	raise RuntimeError("\n%s\n(Hint: copy/paste from the stable branch version of the file(s).)" % "\n".join(errorMessages))


	def normalizeVersion(tup: tuple[str, ...]):
	suffix = ""
	if tup[-1] is not None and tup[-1].lower() == "-alpha":
	tup = tup[: (len(tup) - 1)]
	suffix = "-ALPHA"
	elif tup[-1] is not None and tup[-1].lower() == "-beta":
	tup = tup[: (len(tup) - 1)]
	suffix = "-BETA"
	while tup[-1] in ("", None):
	tup = tup[: (len(tup) - 1)]
	while len(tup) < 3:
	tup = tup + ("0",)
	return ".".join(tup) + suffix


	def pushLocal(version: str, root: str, rcNum: int, localDir: str):
	print("Push local [%s]..." % localDir)
	Path(localDir).mkdir(parents=True)

	lucene_dist_dir = "%s/lucene/distribution/build/release" % root
	rev = open("%s/lucene/distribution/build/release/.gitrev" % root, encoding="UTF-8").read()

	dir = "lucene-%s-RC%d-rev-%s" % (version, rcNum, rev)
	Path("%s/%s/lucene" % (localDir, dir)).mkdir(parents=True)
	print(" Lucene")
	os.chdir(lucene_dist_dir)
	print(" archive...")
	if os.path.exists("lucene.tar"):
	os.remove("lucene.tar")
	run("tar cf lucene.tar *")

	os.chdir("%s/%s/lucene" % (localDir, dir))
	print(" extract...")
	run('tar xf "%s/lucene.tar"' % lucene_dist_dir)
	os.remove("%s/lucene.tar" % lucene_dist_dir)
	os.chdir("..")

	print(" chmod...")
	run("chmod -R a+rX-w .")

	print(" done!")
	return "file://%s/%s" % (Path(localDir).resolve(), dir)


	def read_version(_path: str):
	return scriptutil.find_current_version()


	def parse_config():
	epilogue = textwrap.dedent("""
	Example usage for a Release Manager:
	python3 -u dev-tools/scripts/buildAndPushRelease.py --push-local /tmp/releases/6.0.1 --sign 6E68DA61 --rc-num 1
	""")
	description = "Utility to build, push, and test a release."
	parser = argparse.ArgumentParser(description=description, epilog=epilogue, formatter_class=argparse.RawDescriptionHelpFormatter)
	parser.add_argument("--no-prepare", dest="prepare", default=True, action="store_false", help="Use the already built release in the provided checkout")
	parser.add_argument("--local-keys", metavar="PATH", help="Uses local KEYS file to validate presence of RM's gpg key")
	parser.add_argument("--push-local", metavar="PATH", help="Push the release to the local path")
	parser.add_argument("--pause-before-sign", default=False, action="store_true", help="Pause for user confirmation before the assembleRelease step (to prevent timeout on gpg pinentry")
	parser.add_argument("--sign", metavar="KEYID", help="Sign the release with the given gpg key")
	parser.add_argument(
	"--sign-method-gradle",
	dest="sign_method_gradle",
	default=False,
	action="store_true",
	help="Use Gradle built-in GPG signing instead of gpg command for signing artifacts. This may require --gpg-secring argument if your keychain cannot be resolved automatically.",
	)
	parser.add_argument(
	"--gpg-pass-noprompt",
	dest="gpg_pass_noprompt",
	default=False,
	action="store_true",
	help="""
	Do not prompt for gpg passphrase. For the default gnupg method, this means your gpg-agent
	needs a non-TTY pin-entry program. For gradle signing method, passphrase must be provided
	in gradle.properties or by env.var/sysprop. See ./gradlew helpPublishing for more info
	""".strip(),
	)
	parser.add_argument("--gpg-home", metavar="PATH", help="Path to gpg home containing your secring.gpg Optional, will use $HOME/.gnupg/secring.gpg by default")
	parser.add_argument("--rc-num", metavar="NUM", type=int, default=1, help="Release Candidate number. Default: 1")
	parser.add_argument("--root", metavar="PATH", default=".", help='Root of Git working tree for lucene. Default: "." (the current directory)')
	parser.add_argument("--logfile", metavar="PATH", help="Specify log file path (default /tmp/release.log)")
	parser.add_argument("--dev-mode", default=False, action="store_true", help="Enable development mode, which disables some strict checks")
	config = parser.parse_args()

	if not config.prepare and config.sign:
	parser.error("Cannot sign already built release")
	if config.push_local is not None and os.path.exists(config.push_local):
	parser.error("Cannot push to local path that already exists")
	if config.rc_num <= 0:
	parser.error("Release Candidate number must be a positive integer")
	if not os.path.isdir(config.root):
	parser.error('Root path "%s" is not a directory' % config.root)
	if config.local_keys is not None and not os.path.exists(config.local_keys):
	parser.error('Local KEYS file "%s" not found' % config.local_keys)
	if config.gpg_home and not os.path.exists(os.path.join(config.gpg_home, "secring.gpg")):
	parser.error("Specified gpg home %s does not exist or does not contain a secring.gpg" % config.gpg_home)
	global dev_mode
	if config.dev_mode:
	print("Enabling development mode - DO NOT USE FOR ACTUAL RELEASE!")
	dev_mode = True
	cwd = os.getcwd()
	os.chdir(config.root)
	config.root = os.getcwd() # Absolutize root dir
	if os.system("git rev-parse") or len([d for d in ("dev-tools", "lucene") if os.path.isdir(d)]) != 2:
	parser.error('Root path "%s" is not a valid lucene checkout' % config.root)
	os.chdir(cwd)
	global LOG
	if config.logfile:
	LOG = config.logfile
	print("Logfile is: %s" % LOG)

	config.version = read_version(config.root)
	print("Building version: %s" % config.version)

	return config


	def check_cmdline_tools(): # Fail fast if there are cmdline tool problems
	if os.system("git --version >/dev/null 2>/dev/null"):
	raise RuntimeError('"git --version" returned a non-zero exit code.')


	def check_key_in_keys(gpgKeyID: str \| None, local_keys: str \| None):
	if gpgKeyID is not None:
	print(" Verify your gpg key is in the main KEYS file")
	if local_keys is not None:
	print(" Using local KEYS file %s" % local_keys)
	keysFileText = open(local_keys, encoding="iso-8859-1").read()
	keysFileLocation = local_keys
	else:
	keysFileURL = "https://archive.apache.org/dist/lucene/KEYS"
	keysFileLocation = keysFileURL
	print(" Using online KEYS file %s" % keysFileURL)
	keysFileText = load(keysFileURL, encoding="iso-8859-1")
	if len(gpgKeyID) > 2 and gpgKeyID[0:2] == "0x":
	gpgKeyID = gpgKeyID[2:]
	if len(gpgKeyID) > 40:
	gpgKeyID = gpgKeyID.replace(" ", "")
	if len(gpgKeyID) == 8:
	gpgKeyID8Char = "%s %s" % (gpgKeyID[0:4], gpgKeyID[4:8])
	re_to_match = r"^pub .*\n\s+(\w{4} \w{4} \w{4} \w{4} \w{4} \w{4} \w{4} \w{4} %s\|\w{32}%s)" % (gpgKeyID8Char, gpgKeyID)
	elif len(gpgKeyID) == 40:
	gpgKeyID40Char = "%s %s %s %s %s %s %s %s %s %s" % (
	gpgKeyID[0:4],
	gpgKeyID[4:8],
	gpgKeyID[8:12],
	gpgKeyID[12:16],
	gpgKeyID[16:20],
	gpgKeyID[20:24],
	gpgKeyID[24:28],
	gpgKeyID[28:32],
	gpgKeyID[32:36],
	gpgKeyID[36:],
	)
	re_to_match = r"^pub .*\n\s+(%s\|%s)" % (gpgKeyID40Char, gpgKeyID)
	else:
	print("Invalid gpg key id format [%s]. Must be 8 byte short ID or 40 byte fingerprint, with or without 0x prefix, no spaces." % gpgKeyID)
	exit(2)
	if re.search(re_to_match, keysFileText, re.MULTILINE):
	print(" Found key %s in KEYS file at %s" % (gpgKeyID, keysFileLocation))
	else:
	print(" ERROR: Did not find your key %s in KEYS file at %s. Please add it and try again." % (gpgKeyID, keysFileLocation))
	if local_keys is not None:
	print(" You are using a local KEYS file. Make sure it is up to date or validate against the online version")
	exit(2)


	def resolve_gpghome():
	for p in [
	# Linux, macos
	os.path.join(os.path.expanduser("~"), ".gnupg"),
	# Windows 10
	os.path.expandvars(r"%APPDATA%\GnuPG"),
	# TODO: Should we support Cygwin?
	]:
	if os.path.exists(os.path.join(p, "secring.gpg")):
	return p
	return None


	def main():
	check_cmdline_tools()

	c = parse_config()
	gpg_home = None

	if c.sign:
	sys.stdout.flush()
	c.key_id = c.sign
	check_key_in_keys(c.key_id, c.local_keys)
	if c.gpg_home is not None:
	print("Using custom gpg-home: %s" % c.gpg_home)
	gpg_home = c.gpg_home
	if c.sign_method_gradle:
	if gpg_home is None:
	resolved_gpg_home = resolve_gpghome()
	if resolved_gpg_home is not None:
	print("Resolved gpg home to %s" % resolved_gpg_home)
	gpg_home = resolved_gpg_home
	else:
	print("WARN: Could not locate your gpg secret keyring, and --gpg-home not specified.")
	print(" Falling back to location configured in gradle.properties.")
	print(" See 'gradlew helpPublishing' for details.")
	gpg_home = None
	if c.gpg_pass_noprompt:
	print("Will not prompt for gpg password. Make sure your signing setup supports this.")
	c.key_password = None
	else:
	c.key_password = getpass.getpass("Enter GPG keystore password: ")
	else:
	c.key_id = None
	c.key_password = None

	if c.prepare:
	prepare(c.root, c.version, c.pause_before_sign, c.key_id, c.key_password, gpg_home=gpg_home, sign_gradle=c.sign_method_gradle)
	else:
	os.chdir(c.root)

	if c.push_local:
	url = pushLocal(c.version, c.root, c.rc_num, c.push_local)
	else:
	url = None

	if url is not None:
	print(" URL: %s" % url)
	print("Next run the smoker tester:")
	p = re.compile(".*/")
	m = p.match(sys.argv[0])
	assert m
	if not c.sign:
	signed = "--not-signed"
	else:
	signed = ""
	print("%s -u %ssmokeTestRelease.py %s %s" % (sys.executable, m.group(), signed, url))


	if __name__ == "__main__":
	try:
	main()
	except KeyboardInterrupt:
	print("Keyboard interrupt...exiting")