dev-tools/scripts/addBackcompatIndexes.py

#!/usr/bin/env python3
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


# For usage information, see:
#
#   http://wiki.apache.org/lucene-java/ReleaseTodo#Generate_Backcompat_Indexes


import os
import sys
from pathlib import Path

sys.path.append(os.path.dirname(__file__))
import argparse
import re
import shutil
import urllib.error
import urllib.request

import scriptutil


def create_and_add_index(source: str, indextype: str, index_version: scriptutil.Version, current_version: scriptutil.Version, temp_dir: str):
  if not current_version.is_back_compat_with(index_version):
    prefix = "unsupported"
  else:
    prefix = {"cfs": "index", "nocfs": "index", "sorted": "sorted", "int7_hnsw": "int7_hnsw", "moreterms": "moreterms", "dvupdates": "dvupdates", "emptyIndex": "empty"}[indextype]
  if indextype in ("cfs", "nocfs"):
    filename = "%s.%s-%s.zip" % (prefix, index_version, indextype)
  else:
    filename = "%s.%s.zip" % (prefix, index_version)

  print("  creating %s..." % filename, end="", flush=True)
  module = "backward-codecs"
  index_dir = os.path.join("lucene", module, "src/test/org/apache/lucene/backward_index")
  if os.path.exists(os.path.join(index_dir, filename)):
    print("uptodate")
    return

  test = {
    "cfs": "testCreateCFS",
    "nocfs": "testCreateNoCFS",
    "sorted": "testCreateSortedIndex",
    "int7_hnsw": "testCreateInt7HNSWIndices",
    "moreterms": "testCreateMoreTermsIndex",
    "dvupdates": "testCreateIndexWithDocValuesUpdates",
    "emptyIndex": "testCreateEmptyIndex",
  }[indextype]
  gradle_args = " ".join(["-p lucene/%s" % module, "test", "--tests TestGenerateBwcIndices.%s" % test, "-Dtests.bwcdir=%s" % temp_dir, "-Dtests.codec=default"])
  base_dir = os.getcwd()
  bc_index_file = os.path.join(temp_dir, filename)

  if os.path.exists(bc_index_file):
    print("alreadyexists")
  else:
    os.chdir(source)
    scriptutil.run("./gradlew %s" % gradle_args)
    if not os.path.exists(bc_index_file):
      raise Exception("Expected file can't be found: %s" % bc_index_file)
    print("done")

  print("  adding %s..." % filename, end="", flush=True)
  scriptutil.run("cp %s %s" % (bc_index_file, os.path.join(base_dir, index_dir)))
  os.chdir(base_dir)
  print("done")


def update_backcompat_tests(index_version: scriptutil.Version, current_version: scriptutil.Version):
  print("  adding new indexes to backcompat tests...", end="", flush=True)
  module = "lucene/backward-codecs"

  filename = None
  if not current_version.is_back_compat_with(index_version):
    filename = "%s/src/test/org/apache/lucene/backward_index/unsupported_versions.txt" % module
  else:
    filename = "%s/src/test/org/apache/lucene/backward_index/versions.txt" % module

  strip_dash_suffix_re = re.compile(r"-.*")

  def find_version(x: str):
    x = x.strip()
    x = re.sub(strip_dash_suffix_re, "", x)  # remove the -suffix if any
    return scriptutil.Version.parse(x)

  def edit(buffer: list[str], _match: re.Match[str], line: str):
    v = find_version(line)
    changed = False
    if v.on_or_after(index_version):
      if not index_version.on_or_after(v):
        buffer.append(("%s\n") % index_version)
      changed = True
    buffer.append(line)
    return changed

  def append(buffer: list[str], changed: bool):
    if changed:
      return changed
    if not buffer[len(buffer) - 1].endswith("\n"):
      buffer.append("\n")
    buffer.append(("%s\n") % index_version)
    return True

  changed = scriptutil.update_file(filename, re.compile(r".*"), edit, append)
  print("done" if changed else "uptodate")


def check_backcompat_tests():
  print("  checking backcompat tests...", end="", flush=True)
  scriptutil.run("./gradlew -p lucene/backward-codecs test --tests TestGenerateBwcIndices")
  print("ok")


def download_from_cdn(version: scriptutil.Version, remotename: str, localname: str):
  url = "http://dlcdn.apache.org/lucene/java/%s/%s" % (version, remotename)
  try:
    urllib.request.urlretrieve(url, localname)
    return True
  except urllib.error.HTTPError as e:
    if e.code == 404:
      return False
    raise e


def download_from_archives(version: scriptutil.Version, remotename: str, localname: str):
  url = "http://archive.apache.org/dist/lucene/java/%s/%s" % (version, remotename)
  try:
    urllib.request.urlretrieve(url, localname)
    return True
  except urllib.error.HTTPError as e:
    if e.code == 404:
      return False
    raise e


def download_release(version: scriptutil.Version, temp_dir: str, force: bool):
  print("  downloading %s source release..." % version, end="", flush=True)
  source = os.path.join(temp_dir, "lucene-%s" % version)
  if os.path.exists(source):
    if force:
      shutil.rmtree(source)
    else:
      print("uptodate")
      return source

  filename = "lucene-%s-src.tgz" % version
  source_tgz = os.path.join(temp_dir, filename)
  if not download_from_cdn(version, filename, source_tgz) and not download_from_archives(version, filename, source_tgz):
    raise Exception("Could not find version %s in apache CDN or archives" % version)

  olddir = os.getcwd()
  os.chdir(temp_dir)
  scriptutil.run("tar -xvzf %s" % source_tgz)
  os.chdir(olddir)
  print("done")
  return source


def read_config():
  parser = argparse.ArgumentParser(
    formatter_class=argparse.RawDescriptionHelpFormatter,
    description="""\
Add backcompat index and test for new version.  See:
http://wiki.apache.org/lucene-java/ReleaseTodo#Generate_Backcompat_Indexes
""",
  )
  parser.add_argument("--force", action="store_true", default=False, help="Redownload the version and rebuild, even if it already exists")
  parser.add_argument("--no-cleanup", dest="cleanup", action="store_false", default=True, help="Do not cleanup the built indexes, so that they can be reused " + "for adding to another branch")
  parser.add_argument("--temp-dir", metavar="DIR", default="/tmp/lucenebwc", help="Temp directory to build backcompat indexes within")
  parser.add_argument("version", type=scriptutil.Version.parse, help="Version to add, of the form X.Y.Z")
  c = parser.parse_args()

  return c


def main():
  c = read_config()
  if not os.path.exists(c.temp_dir):
    Path(c.temp_dir).mkdir(parents=True)

  print("\nCreating backwards compatibility indexes")
  source = download_release(c.version, c.temp_dir, c.force)
  current_version = scriptutil.Version.parse(scriptutil.find_current_version())
  create_and_add_index(source, "cfs", c.version, current_version, c.temp_dir)
  create_and_add_index(source, "nocfs", c.version, current_version, c.temp_dir)
  create_and_add_index(source, "int7_hnsw", c.version, current_version, c.temp_dir)
  should_make_sorted = current_version.is_back_compat_with(c.version) and (c.version.major > 6 or (c.version.major == 6 and c.version.minor >= 2))
  if should_make_sorted:
    create_and_add_index(source, "sorted", c.version, current_version, c.temp_dir)
  if c.version.minor == 0 and c.version.bugfix == 0 and current_version.is_back_compat_with(c.version):
    create_and_add_index(source, "moreterms", c.version, current_version, c.temp_dir)
    create_and_add_index(source, "dvupdates", c.version, current_version, c.temp_dir)
    create_and_add_index(source, "emptyIndex", c.version, current_version, c.temp_dir)
    print("\nMANUAL UPDATE REQUIRED: edit TestGenerateBwcIndices to enable moreterms, dvupdates, and empty index testing")

  print("\nAdding backwards compatibility tests")
  update_backcompat_tests(c.version, current_version)

  print("\nTesting changes")
  check_backcompat_tests()

  if c.cleanup:
    print("\nCleaning up")
    print("  deleting %s..." % c.temp_dir, end="", flush=True)
    shutil.rmtree(c.temp_dir)
    print("done")

  print()


if __name__ == "__main__":
  try:
    main()
  except KeyboardInterrupt:
    print("\nRecieved Ctrl-C, exiting early")