HDDS-4741. Modularize upgrade test (#1928)

diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-mr/test.sh
index c520348..4e0151b 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/test.sh
@@ -17,7 +17,7 @@
 SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )
 ALL_RESULT_DIR="$SCRIPT_DIR/result"
 mkdir -p "$ALL_RESULT_DIR"
-rm "$ALL_RESULT_DIR/*" || true
+rm "$ALL_RESULT_DIR"/* || true
 source "$SCRIPT_DIR/../testlib.sh"
 
 tests=$(find_tests)
diff --git a/hadoop-ozone/dist/src/main/compose/restart/test.sh b/hadoop-ozone/dist/src/main/compose/restart/test.sh
index 41e06c5..cf0f532 100644
--- a/hadoop-ozone/dist/src/main/compose/restart/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/restart/test.sh
@@ -23,19 +23,12 @@
 
 export OZONE_VOLUME
 
-mkdir -p "${OZONE_VOLUME}"/{dn1,dn2,dn3,om,recon,s3g,scm}
-
-if [[ -n "${OZONE_VOLUME_OWNER}" ]]; then
-  current_user=$(whoami)
-  if [[ "${OZONE_VOLUME_OWNER}" != "${current_user}" ]]; then
-    chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}" \
-      || sudo chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}"
-  fi
-fi
-
 # shellcheck source=/dev/null
 source "${COMPOSE_DIR}/../testlib.sh"
 
+mkdir -p "${OZONE_VOLUME}"/{dn1,dn2,dn3,om,recon,s3g,scm}
+fix_data_dir_permissions
+
 # prepare pre-upgrade cluster
 start_docker_env
 execute_robot_test scm -v PREFIX:pre freon/generate.robot
diff --git a/hadoop-ozone/dist/src/main/compose/test-all.sh b/hadoop-ozone/dist/src/main/compose/test-all.sh
index c51a969..3ef2dcf 100755
--- a/hadoop-ozone/dist/src/main/compose/test-all.sh
+++ b/hadoop-ozone/dist/src/main/compose/test-all.sh
@@ -23,7 +23,7 @@
 ALL_RESULT_DIR="$SCRIPT_DIR/result"
 PROJECT_DIR="$SCRIPT_DIR/.."
 mkdir -p "$ALL_RESULT_DIR"
-rm "$ALL_RESULT_DIR/*" || true
+rm "$ALL_RESULT_DIR"/* || true
 
 source "$SCRIPT_DIR"/testlib.sh
 
diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh
index 0e12fde..3ba5bcd 100755
--- a/hadoop-ozone/dist/src/main/compose/testlib.sh
+++ b/hadoop-ozone/dist/src/main/compose/testlib.sh
@@ -16,6 +16,9 @@
 # limitations under the License.
 set -e
 
+_testlib_this="${BASH_SOURCE[0]}"
+_testlib_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
 COMPOSE_ENV_NAME=$(basename "$COMPOSE_DIR")
 RESULT_DIR=${RESULT_DIR:-"$COMPOSE_DIR/result"}
 RESULT_DIR_INSIDE="/tmp/smoketest/$(basename "$COMPOSE_ENV_NAME")/result"
@@ -280,12 +283,14 @@
 
 ## @description  Generate robot framework reports based on the saved results.
 generate_report(){
+  local title="${1:-${COMPOSE_ENV_NAME}}"
+  local dir="${2:-${RESULT_DIR}}"
 
   if command -v rebot > /dev/null 2>&1; then
      #Generate the combined output and return with the right exit code (note: robot = execute test, rebot = generate output)
-     rebot --reporttitle "${COMPOSE_ENV_NAME}" -N "${COMPOSE_ENV_NAME}" -d "$RESULT_DIR" "$RESULT_DIR/robot-*.xml"
+     rebot --reporttitle "${title}" -N "${title}" -d "${dir}" "${dir}/*.xml"
   else
-     echo "Robot framework is not installed, the reports can be generated (sudo pip install robotframework)."
+     echo "Robot framework is not installed, the reports cannot be generated (sudo pip install robotframework)."
      exit 1
   fi
 }
@@ -298,7 +303,7 @@
   local result_dir="${test_dir}/result"
   local test_dir_name=$(basename ${test_dir})
   if [[ -n "$(find "${result_dir}" -name "*.xml")" ]]; then
-    rebot --nostatusrc -N "${test_dir_name}" -o "${all_result_dir}/${test_dir_name}.xml" "${result_dir}/*.xml"
+    rebot --nostatusrc -N "${test_dir_name}" -l NONE -r NONE -o "${all_result_dir}/${test_dir_name}.xml" "${result_dir}/*.xml"
   fi
 
   cp "${result_dir}"/docker-*.log "${all_result_dir}"/
@@ -325,3 +330,63 @@
 
   return ${ret}
 }
+
+## @description Make `OZONE_VOLUME_OWNER` the owner of the `OZONE_VOLUME`
+##   directory tree (required in Github Actions runner environment)
+fix_data_dir_permissions() {
+  if [[ -n "${OZONE_VOLUME}" ]] && [[ -n "${OZONE_VOLUME_OWNER}" ]]; then
+    current_user=$(whoami)
+    if [[ "${OZONE_VOLUME_OWNER}" != "${current_user}" ]]; then
+      chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}" \
+        || sudo chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}"
+    fi
+  fi
+}
+
+## @description Define variables required for using Ozone docker image which
+##   includes binaries for a specific release
+## @param `ozone` image version
+prepare_for_binary_image() {
+  local v=$1
+
+  export OZONE_DIR=/opt/ozone
+  export OZONE_IMAGE="apache/ozone:${v}"
+}
+
+## @description Define variables required for using `ozone-runner` docker image
+##   (no binaries included)
+## @param `ozone-runner` image version (optional)
+prepare_for_runner_image() {
+  local default_version=${docker.ozone-runner.version} # set at build-time from Maven property
+  local runner_version=${OZONE_RUNNER_VERSION:-${default_version}} # may be specified by user running the test
+  local v=${1:-${runner_version}} # prefer explicit argument
+
+  export OZONE_DIR=/opt/hadoop
+  export OZONE_IMAGE="apache/ozone-runner:${v}"
+}
+
+## @description Print the logical version for a specific release
+## @param the release for which logical version should be printed
+get_logical_version() {
+  local v="$1"
+
+  # shellcheck source=/dev/null
+  echo $(source "${_testlib_dir}/versions/${v}.sh" && ozone_logical_version)
+}
+
+## @description Activate the version-specific behavior for a given release
+## @param the release for which definitions should be loaded
+load_version_specifics() {
+  local v="$1"
+
+  # shellcheck source=/dev/null
+  source "${_testlib_dir}/versions/${v}.sh"
+
+  ozone_version_load
+}
+
+## @description Deactivate the previously version-specific behavior,
+##   reverting to the current version's definitions
+unload_version_specifics() {
+  ozone_version_unload
+}
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.5.0.sh b/hadoop-ozone/dist/src/main/compose/upgrade/0.5.0/test.sh
old mode 100644
new mode 100755
similarity index 78%
rename from hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.5.0.sh
rename to hadoop-ozone/dist/src/main/compose/upgrade/0.5.0/test.sh
index 667ce95..11530c7
--- a/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.5.0.sh
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/0.5.0/test.sh
@@ -15,5 +15,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-export OZONE_ADMIN_COMMAND=scmcli
-export OZONE_SAFEMODE_STATUS_COMMAND='ozone scmcli safemode status'
+_mydir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+export OZONE_UPGRADE_FROM="$(basename ${_mydir})"
+export RESULT_DIR="${_mydir}/result"
+
+cd "${_mydir}/.."
+source upgrade_to_current.sh
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.5.0.sh b/hadoop-ozone/dist/src/main/compose/upgrade/1.0.0/test.sh
old mode 100644
new mode 100755
similarity index 78%
copy from hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.5.0.sh
copy to hadoop-ozone/dist/src/main/compose/upgrade/1.0.0/test.sh
index 667ce95..11530c7
--- a/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.5.0.sh
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/1.0.0/test.sh
@@ -15,5 +15,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-export OZONE_ADMIN_COMMAND=scmcli
-export OZONE_SAFEMODE_STATUS_COMMAND='ozone scmcli safemode status'
+_mydir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+export OZONE_UPGRADE_FROM="$(basename ${_mydir})"
+export RESULT_DIR="${_mydir}/result"
+
+cd "${_mydir}/.."
+source upgrade_to_current.sh
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
old mode 100644
new mode 100755
index 1c16c81..929be05
--- a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
@@ -15,59 +15,27 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-export COMPOSE_DIR
+SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )
+ALL_RESULT_DIR="$SCRIPT_DIR/result"
+mkdir -p "$ALL_RESULT_DIR"
+rm "$ALL_RESULT_DIR"/* || true
+source "$SCRIPT_DIR/../testlib.sh"
 
-: "${OZONE_REPLICATION_FACTOR:=3}"
-: "${OZONE_UPGRADE_FROM:="0.5.0"}"
-: "${OZONE_UPGRADE_TO:="1.0.0"}"
-: "${OZONE_VOLUME:="${COMPOSE_DIR}/data"}"
+tests=$(find_tests)
+cd "$SCRIPT_DIR"
 
-export OZONE_VOLUME
+RESULT=0
+# shellcheck disable=SC2044
+for t in ${tests}; do
+  d="$(dirname "${t}")"
 
-mkdir -p "${OZONE_VOLUME}"/{dn1,dn2,dn3,om,recon,s3g,scm}
-
-if [[ -n "${OZONE_VOLUME_OWNER}" ]]; then
-  current_user=$(whoami)
-  if [[ "${OZONE_VOLUME_OWNER}" != "${current_user}" ]]; then
-    chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}" \
-      || sudo chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}"
+  if ! run_test_script "${d}"; then
+    RESULT=1
   fi
-fi
 
-# define version-specifics
-export OZONE_DIR=/opt/ozone
-export OZONE_IMAGE="apache/ozone:${OZONE_UPGRADE_FROM}"
-# shellcheck source=/dev/null
-source "${COMPOSE_DIR}/versions/ozone-${OZONE_UPGRADE_FROM}.sh"
-# shellcheck source=/dev/null
-source "${COMPOSE_DIR}/../testlib.sh"
+  copy_results "${d}" "${ALL_RESULT_DIR}"
+done
 
-# prepare pre-upgrade cluster
-start_docker_env
-execute_robot_test scm -v PREFIX:pre freon/generate.robot
-execute_robot_test scm -v PREFIX:pre freon/validate.robot
-KEEP_RUNNING=false stop_docker_env
+generate_report "upgrade" "${ALL_RESULT_DIR}"
 
-# run upgrade scripts
-SCRIPT_DIR=../../libexec/upgrade
-[[ -f "${SCRIPT_DIR}/${OZONE_UPGRADE_TO}.sh" ]] && "${SCRIPT_DIR}/${OZONE_UPGRADE_TO}.sh"
-
-# update version-specifics
-export OZONE_DIR=/opt/hadoop
-unset OZONE_IMAGE # use apache/ozone-runner defined in docker-compose.yaml
-# shellcheck source=/dev/null
-source "${COMPOSE_DIR}/versions/ozone-${OZONE_UPGRADE_TO}.sh"
-# shellcheck source=/dev/null
-source "${COMPOSE_DIR}/../testlib.sh"
-
-# re-start cluster with new version and check after upgrade
-export OZONE_KEEP_RESULTS=true
-start_docker_env
-execute_robot_test scm -v PREFIX:pre freon/validate.robot
-# test write key to old bucket after upgrade
-execute_robot_test scm -v PREFIX:post freon/generate.robot
-execute_robot_test scm -v PREFIX:post freon/validate.robot
-stop_docker_env
-
-generate_report
+exit ${RESULT}
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/testlib.sh b/hadoop-ozone/dist/src/main/compose/upgrade/testlib.sh
new file mode 100755
index 0000000..1c6cfe6
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/testlib.sh
@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e -o pipefail
+
+_upgrade_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+: "${OZONE_REPLICATION_FACTOR:=3}"
+: "${OZONE_VOLUME:="${COMPOSE_DIR}/data"}"
+: "${OZONE_VOLUME_OWNER:=}"
+
+source "${_upgrade_dir}/../testlib.sh"
+
+## @description Create the directory tree required for persisting data between
+##   compose cluster restarts
+create_data_dir() {
+  if [[ -z "${OZONE_VOLUME}" ]]; then
+    return 1
+  fi
+
+  rm -fr "${OZONE_VOLUME}" 2> /dev/null || sudo rm -fr "${OZONE_VOLUME}"
+  mkdir -p "${OZONE_VOLUME}"/{dn1,dn2,dn3,om,recon,s3g,scm}
+  fix_data_dir_permissions
+}
+
+## @description Run upgrade steps required for going from one logical version to another.
+## @param Starting logical version
+## @param Target logical version
+execute_upgrade_steps() {
+  local -i from=$1
+  local -i to=$2
+
+  if [[ ${from} -ge ${to} ]]; then
+    return
+  fi
+
+  pushd ${_testlib_dir}/../libexec/upgrade
+
+  local v
+  for v in $(seq ${from} $((to-1))); do
+    if [[ -e "v$v.sh" ]]; then
+      source "v$v.sh"
+    fi
+  done
+
+  popd
+}
+
+## @description Pre-upgrade test steps
+first_run() {
+  start_docker_env
+  execute_robot_test scm -v PREFIX:pre freon/generate.robot
+  execute_robot_test scm -v PREFIX:pre freon/validate.robot
+  KEEP_RUNNING=false stop_docker_env
+}
+
+## @description Post-upgrade test steps
+second_run() {
+  export OZONE_KEEP_RESULTS=true
+  start_docker_env
+  execute_robot_test scm -v PREFIX:pre freon/validate.robot
+  # test write key to old bucket after upgrade
+  execute_robot_test scm -v PREFIX:post freon/generate.robot
+  execute_robot_test scm -v PREFIX:post freon/validate.robot
+  stop_docker_env
+}
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/upgrade_to_current.sh b/hadoop-ozone/dist/src/main/compose/upgrade/upgrade_to_current.sh
new file mode 100755
index 0000000..8728851
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/upgrade_to_current.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script tests upgrade from a previous release to the current
+# binaries.  Docker image with Ozone binaries is required for the
+# initial version, while the snapshot version uses Ozone runner image.
+
+set -e -o pipefail
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+: "${OZONE_REPLICATION_FACTOR:=3}"
+: "${OZONE_UPGRADE_FROM:="0.5.0"}"
+: "${OZONE_VOLUME:="${COMPOSE_DIR}/data"}"
+
+export OZONE_REPLICATION_FACTOR OZONE_UPGRADE_FROM OZONE_VOLUME
+
+current_version=1.1.0
+
+source "${COMPOSE_DIR}/testlib.sh"
+
+create_data_dir
+
+prepare_for_binary_image "${OZONE_UPGRADE_FROM}"
+load_version_specifics "${OZONE_UPGRADE_FROM}"
+first_run
+unload_version_specifics
+
+from=$(get_logical_version "${OZONE_UPGRADE_FROM}")
+to=$(get_logical_version "${current_version}")
+execute_upgrade_steps "$from" "$to"
+
+prepare_for_runner_image
+second_run
+
+generate_report
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/upgrade_to_release.sh b/hadoop-ozone/dist/src/main/compose/upgrade/upgrade_to_release.sh
new file mode 100755
index 0000000..5a9b402
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/upgrade_to_release.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script tests upgrade from one release to a later one.  Docker
+# image with Ozone binaries are required for both versions.
+
+set -e -o pipefail
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+: "${OZONE_REPLICATION_FACTOR:=3}"
+: "${OZONE_UPGRADE_FROM:="0.5.0"}"
+: "${OZONE_UPGRADE_TO:="1.0.0"}"
+: "${OZONE_VOLUME:="${COMPOSE_DIR}/data"}"
+
+export OZONE_REPLICATION_FACTOR OZONE_UPGRADE_FROM OZONE_UPGRADE_TO OZONE_VOLUME
+
+source "${COMPOSE_DIR}/testlib.sh"
+
+create_data_dir
+
+prepare_for_binary_image "${OZONE_UPGRADE_FROM}"
+load_version_specifics "${OZONE_UPGRADE_FROM}"
+first_run
+unload_version_specifics
+
+from=$(get_logical_version "${OZONE_UPGRADE_FROM}")
+to=$(get_logical_version "${OZONE_UPGRADE_TO}")
+execute_upgrade_steps "$from" "$to"
+
+prepare_for_binary_image "${OZONE_UPGRADE_TO}"
+load_version_specifics "${OZONE_UPGRADE_TO}"
+second_run
+unload_version_specifics
+
+generate_report
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/versions/README.md b/hadoop-ozone/dist/src/main/compose/upgrade/versions/README.md
deleted file mode 100644
index 24cd113..0000000
--- a/hadoop-ozone/dist/src/main/compose/upgrade/versions/README.md
+++ /dev/null
@@ -1,15 +0,0 @@
-<!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
--->
-
-The scripts in this directory define version-specific behavior required for [`testlib.sh`](../../testlib.sh).  For example the `ozone admin` command was renamed from `ozone scmcli` in 1.0.0.
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-1.0.0.sh b/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-1.0.0.sh
deleted file mode 100644
index 3ff23e0..0000000
--- a/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-1.0.0.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-export OZONE_ADMIN_COMMAND=admin
-export OZONE_SAFEMODE_STATUS_COMMAND='ozone admin safemode status --verbose'
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.5.0.sh b/hadoop-ozone/dist/src/main/compose/versions/0.5.0.sh
similarity index 75%
copy from hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.5.0.sh
copy to hadoop-ozone/dist/src/main/compose/versions/0.5.0.sh
index 667ce95..f973a04 100644
--- a/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.5.0.sh
+++ b/hadoop-ozone/dist/src/main/compose/versions/0.5.0.sh
@@ -15,5 +15,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-export OZONE_ADMIN_COMMAND=scmcli
-export OZONE_SAFEMODE_STATUS_COMMAND='ozone scmcli safemode status'
+ozone_logical_version() {
+  echo 1
+}
+
+ozone_version_load() {
+  export OZONE_ADMIN_COMMAND=scmcli
+  export OZONE_SAFEMODE_STATUS_COMMAND='ozone scmcli safemode status'
+}
+
+ozone_version_unload() {
+  unset OZONE_ADMIN_COMMAND
+  unset OZONE_SAFEMODE_STATUS_COMMAND
+}
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.5.0.sh b/hadoop-ozone/dist/src/main/compose/versions/1.0.0.sh
similarity index 74%
copy from hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.5.0.sh
copy to hadoop-ozone/dist/src/main/compose/versions/1.0.0.sh
index 667ce95..c56aa33 100644
--- a/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.5.0.sh
+++ b/hadoop-ozone/dist/src/main/compose/versions/1.0.0.sh
@@ -15,5 +15,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-export OZONE_ADMIN_COMMAND=scmcli
-export OZONE_SAFEMODE_STATUS_COMMAND='ozone scmcli safemode status'
+ozone_logical_version() {
+  echo 2
+}
+
+ozone_version_load() {
+  export OZONE_ADMIN_COMMAND=admin
+  export OZONE_SAFEMODE_STATUS_COMMAND='ozone admin safemode status --verbose'
+}
+
+ozone_version_unload() {
+  unset OZONE_ADMIN_COMMAND
+  unset OZONE_SAFEMODE_STATUS_COMMAND
+}
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.5.0.sh b/hadoop-ozone/dist/src/main/compose/versions/1.1.0.sh
similarity index 74%
copy from hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.5.0.sh
copy to hadoop-ozone/dist/src/main/compose/versions/1.1.0.sh
index 667ce95..c56aa33 100644
--- a/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.5.0.sh
+++ b/hadoop-ozone/dist/src/main/compose/versions/1.1.0.sh
@@ -15,5 +15,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-export OZONE_ADMIN_COMMAND=scmcli
-export OZONE_SAFEMODE_STATUS_COMMAND='ozone scmcli safemode status'
+ozone_logical_version() {
+  echo 2
+}
+
+ozone_version_load() {
+  export OZONE_ADMIN_COMMAND=admin
+  export OZONE_SAFEMODE_STATUS_COMMAND='ozone admin safemode status --verbose'
+}
+
+ozone_version_unload() {
+  unset OZONE_ADMIN_COMMAND
+  unset OZONE_SAFEMODE_STATUS_COMMAND
+}
diff --git a/hadoop-ozone/dist/src/main/compose/versions/README.md b/hadoop-ozone/dist/src/main/compose/versions/README.md
new file mode 100644
index 0000000..f6dc62c
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/versions/README.md
@@ -0,0 +1,23 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+The scripts in this directory define version-specific behavior required for [`testlib.sh`](../testlib.sh).  For example the `ozone admin` command was renamed from `ozone scmcli` in 1.0.0.
+
+For each release a logical version is defined, starting from 1.  It is incremented only if the new version needs to execute some scripts to handle some (unintended) incompatibility.  For example 1.0.0 or later need to rename some RocksDB column-families when upgrading from 0.5.0.
+
+Interface:
+
+ * `ozone_logical_version`: print the logical version
+ * `ozone_version_load`: define version-specific variables for the test library
+ * `ozone_version_unload`: unset version-specific variables; this reverts test library behavior to the "current" one.
diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/test.sh b/hadoop-ozone/dist/src/main/compose/xcompat/test.sh
index 27c5ef4..7d7ee1d 100755
--- a/hadoop-ozone/dist/src/main/compose/xcompat/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/xcompat/test.sh
@@ -21,7 +21,7 @@
 
 current_version=1.1.0
 
-# shellcheck source=/dev/null
+# shellcheck source=hadoop-ozone/dist/src/main/compose/testlib.sh
 source "${COMPOSE_DIR}/../testlib.sh"
 
 old_client() {
@@ -81,13 +81,12 @@
 COMPOSE_FILE=new-cluster.yaml:clients.yaml cluster_version=${current_version} test_cross_compatibility
 
 for cluster_version in 1.0.0; do
-  # shellcheck source=/dev/null
-  source "${COMPOSE_DIR}/../upgrade/versions/ozone-${cluster_version}.sh"
-  # shellcheck source=/dev/null
-  source "${COMPOSE_DIR}/../testlib.sh"
+  load_version_specifics ${cluster_version}
 
   export OZONE_VERSION=${cluster_version}
   COMPOSE_FILE=old-cluster.yaml:clients.yaml test_cross_compatibility
+
+  unload_version_specifics
 done
 
 generate_report
diff --git a/hadoop-ozone/dist/src/main/k8s/examples/test-all.sh b/hadoop-ozone/dist/src/main/k8s/examples/test-all.sh
index ae810c9..5d86651 100755
--- a/hadoop-ozone/dist/src/main/k8s/examples/test-all.sh
+++ b/hadoop-ozone/dist/src/main/k8s/examples/test-all.sh
@@ -24,7 +24,7 @@
 set -ex
 
 ALL_RESULT_DIR="$SCRIPT_DIR/result"
-rm "$ALL_RESULT_DIR/*" || true
+rm "$ALL_RESULT_DIR"/* || true
 mkdir -p "$ALL_RESULT_DIR"
 
 RESULT=0
diff --git a/hadoop-ozone/dist/src/shell/upgrade/1.0.0.sh b/hadoop-ozone/dist/src/shell/upgrade/v1.sh
similarity index 85%
rename from hadoop-ozone/dist/src/shell/upgrade/1.0.0.sh
rename to hadoop-ozone/dist/src/shell/upgrade/v1.sh
index 6573978..1442ffd 100755
--- a/hadoop-ozone/dist/src/shell/upgrade/1.0.0.sh
+++ b/hadoop-ozone/dist/src/shell/upgrade/v1.sh
@@ -20,4 +20,4 @@
 : "${SCM_DIR:="${OZONE_VOLUME}/scm"}"
 : "${OZONE_RUNNER_VERSION:="20200625-1"}"
 
-docker run --rm -v "${SCM_DIR}":/scm -v "${SCRIPT_DIR}/1.0.0":/upgrade -w /scm/metadata apache/ozone-runner:"${OZONE_RUNNER_VERSION}" /upgrade/01-migrate-scm-db.sh
+docker run --rm -v "${SCM_DIR}":/scm -v "${SCRIPT_DIR}/v1":/upgrade -w /scm/metadata apache/ozone-runner:"${OZONE_RUNNER_VERSION}" /upgrade/01-migrate-scm-db.sh
diff --git a/hadoop-ozone/dist/src/shell/upgrade/1.0.0/01-migrate-scm-db.sh b/hadoop-ozone/dist/src/shell/upgrade/v1/01-migrate-scm-db.sh
similarity index 100%
rename from hadoop-ozone/dist/src/shell/upgrade/1.0.0/01-migrate-scm-db.sh
rename to hadoop-ozone/dist/src/shell/upgrade/v1/01-migrate-scm-db.sh