Merge remote-tracking branch 'origin/HDDS-10685' into master
diff --git a/.asf.yaml b/.asf.yaml index 0a4a8f9..bfb637f 100644 --- a/.asf.yaml +++ b/.asf.yaml
@@ -40,7 +40,6 @@ copilot_code_review: enabled: true review_drafts: true - review_on_push: true rulesets: - name: "Default Branch Protection" type: branch
diff --git a/.github/workflows/asf-allowlist-check.yaml b/.github/workflows/asf-allowlist-check.yaml index dfbcec3..1643477 100644 --- a/.github/workflows/asf-allowlist-check.yaml +++ b/.github/workflows/asf-allowlist-check.yaml
@@ -40,7 +40,7 @@ runs-on: ubuntu-slim steps: - name: Checkout project - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Check actions
diff --git a/.github/workflows/build-ratis.yml b/.github/workflows/build-ratis.yml index 4026a53..1bbd8bc 100644 --- a/.github/workflows/build-ratis.yml +++ b/.github/workflows/build-ratis.yml
@@ -69,7 +69,7 @@ thirdparty-version: ${{ steps.versions.outputs.thirdparty }} steps: - name: Checkout project - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false repository: ${{ inputs.repo }} @@ -82,7 +82,7 @@ !~/.m2/repository/org/apache/ratis key: ratis-dependencies-${{ hashFiles('**/pom.xml') }} - name: Setup java - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 + uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0 with: distribution: 'temurin' java-version: 25 @@ -118,7 +118,7 @@ protobuf-version: ${{ steps.versions.outputs.protobuf }} steps: - name: Checkout project - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false repository: apache/ratis-thirdparty
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index ab1493b..5ff9e3b 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml
@@ -153,7 +153,7 @@ steps: - name: Checkout project if: ${{ !inputs.needs-ozone-source-tarball }} - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false ref: ${{ inputs.sha }} @@ -223,7 +223,7 @@ - name: Setup java ${{ inputs.java-version }} if: ${{ inputs.java-version }} - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 + uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0 with: distribution: 'temurin' java-version: ${{ inputs.java-version }}
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02d7414..0243a62 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml
@@ -70,19 +70,19 @@ with-coverage: ${{ env.OZONE_WITH_COVERAGE }} steps: - name: "Checkout ${{ github.ref }} / ${{ github.sha }} (push)" - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false if: github.event_name == 'push' - name: "Checkout ${{ github.sha }} with its parent (pull request)" - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.sha }} fetch-depth: 2 persist-credentials: false if: github.event_name == 'pull_request' - name: "Checkout ${{ inputs.ref }} given in workflow input (manual dispatch)" - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ inputs.ref }} persist-credentials: false @@ -360,7 +360,7 @@ - integration steps: - name: Checkout project - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false @@ -383,7 +383,7 @@ mkdir -p hadoop-ozone/dist/target tar xzvf target/artifacts/ozone-bin/ozone*.tar.gz -C hadoop-ozone/dist/target - name: Setup java ${{ env.TEST_JAVA_VERSION }} - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 + uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0 with: distribution: 'temurin' java-version: ${{ env.TEST_JAVA_VERSION }}
diff --git a/.github/workflows/close-stale-prs.yaml b/.github/workflows/close-stale-prs.yaml index 724176b..25d847b 100644 --- a/.github/workflows/close-stale-prs.yaml +++ b/.github/workflows/close-stale-prs.yaml
@@ -30,6 +30,7 @@ uses: actions/stale@1e223db275d687790206a7acac4d1a11bd6fe629 # v10.4.0 with: stale-pr-label: 'stale' + exempt-pr-labels: 'design' exempt-draft-pr: false days-before-issue-stale: -1 days-before-pr-stale: 21
diff --git a/.github/workflows/generate-config-doc.yml b/.github/workflows/generate-config-doc.yml index 5e73a96..a0451a7 100644 --- a/.github/workflows/generate-config-doc.yml +++ b/.github/workflows/generate-config-doc.yml
@@ -28,13 +28,13 @@ runs-on: ubuntu-slim steps: - name: Checkout project - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false ref: ${{ inputs.sha }} - name: Set up Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: '3.x'
diff --git a/.github/workflows/intermittent-test-check.yml b/.github/workflows/intermittent-test-check.yml index 9734c57..37f1f93 100644 --- a/.github/workflows/intermittent-test-check.yml +++ b/.github/workflows/intermittent-test-check.yml
@@ -78,7 +78,7 @@ outputs: matrix: ${{steps.generate.outputs.matrix}} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false ref: ${{ github.event.inputs.ref }} @@ -107,7 +107,7 @@ timeout-minutes: 60 steps: - name: Checkout project - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false ref: ${{ github.event.inputs.ref }} @@ -128,7 +128,7 @@ path: | ~/.m2/repository/org/apache/ratis - name: Setup java - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 + uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0 with: distribution: 'temurin' java-version: ${{ github.event.inputs.java-version }} @@ -161,7 +161,7 @@ split: ${{fromJson(needs.prepare-job.outputs.matrix)}} # Define splits fail-fast: ${{ fromJson(github.event.inputs.fail-fast) }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false ref: ${{ github.event.inputs.ref }} @@ -189,7 +189,7 @@ path: | ~/.m2/repository/org/apache/ozone - name: Setup java - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 + uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0 with: distribution: 'temurin' java-version: ${{ github.event.inputs.java-version }}
diff --git a/.github/workflows/label-pr.yml b/.github/workflows/label-pr.yml index b5b71d7..2792a5a 100644 --- a/.github/workflows/label-pr.yml +++ b/.github/workflows/label-pr.yml
@@ -37,7 +37,7 @@ fail-fast: false steps: - name: "Checkout project" # required for `gh` CLI - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false sparse-checkout: |
diff --git a/.github/workflows/populate-cache.yml b/.github/workflows/populate-cache.yml index 1789178..ea0939e 100644 --- a/.github/workflows/populate-cache.yml +++ b/.github/workflows/populate-cache.yml
@@ -45,7 +45,7 @@ runs-on: ubuntu-24.04 steps: - name: Checkout project - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -60,7 +60,7 @@ - name: Setup Java if: steps.restore-cache.outputs.cache-hit != 'true' - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 + uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0 with: distribution: 'temurin' java-version: ${{ env.TEST_JAVA_VERSION }} @@ -90,7 +90,7 @@ - name: Setup Java 8 if: steps.restore-cache.outputs.cache-hit != 'true' - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 + uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0 with: distribution: 'temurin' java-version: 8
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 83e2ec0..0dc2c04 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml
@@ -34,7 +34,7 @@ runs-on: ubuntu-slim steps: - name: Checkout project - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Check pull request title
diff --git a/.github/workflows/repeat-acceptance.yml b/.github/workflows/repeat-acceptance.yml index 2368f0c..3f8ec9b 100644 --- a/.github/workflows/repeat-acceptance.yml +++ b/.github/workflows/repeat-acceptance.yml
@@ -57,7 +57,7 @@ outputs: matrix: ${{steps.generate.outputs.matrix}} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false ref: ${{ github.event.inputs.ref }} @@ -83,7 +83,7 @@ timeout-minutes: 60 steps: - name: Checkout project - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false ref: ${{ github.event.inputs.ref }} @@ -107,7 +107,7 @@ maven-repo-${{ hashFiles('**/pom.xml') }} maven-repo- - name: Setup java - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 + uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0 with: distribution: 'temurin' java-version: ${{ env.JAVA_VERSION }} @@ -134,7 +134,7 @@ split: ${{ fromJson(needs.prepare-job.outputs.matrix) }} fail-fast: ${{ fromJson(github.event.inputs.fail-fast) }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false ref: ${{ github.event.inputs.ref }}
diff --git a/.github/workflows/update-ozone-site-config-doc.yml b/.github/workflows/update-ozone-site-config-doc.yml index 2fd6167..5bc9ea2 100644 --- a/.github/workflows/update-ozone-site-config-doc.yml +++ b/.github/workflows/update-ozone-site-config-doc.yml
@@ -53,21 +53,35 @@ - name: Checkout ozone-site repository if: steps.check-site-repo.outputs.exists == 'true' + env: + GH_TOKEN: ${{ secrets.OZONE_WEBSITE_BUILD }} run: | git config --global url."https://asf-ci-deploy:${{ secrets.OZONE_WEBSITE_BUILD }}@github.com/".insteadOf "https://github.com/" git config --global user.name 'github-actions[bot]' git config --global user.email 'github-actions[bot]@users.noreply.github.com' - git clone --depth=1 --branch=master https://github.com/$REPO_OWNER/ozone-site.git ozone-site - - # Check if $BRANCH_NAME branch exists remotely - cd ozone-site - if git ls-remote --heads origin $BRANCH_NAME | grep -q $BRANCH_NAME; then - echo "PR branch exists, checking it out for comparison" - git fetch --depth=1 origin $BRANCH_NAME - git checkout -B $BRANCH_NAME FETCH_HEAD + BRANCH_EXISTS="" + if git clone --depth=1 --branch="$BRANCH_NAME" https://github.com/$REPO_OWNER/ozone-site.git ozone-site; then + BRANCH_EXISTS="true" else - echo "PR branch does not exist, staying on master for comparison" + git clone --depth=1 --branch=master https://github.com/$REPO_OWNER/ozone-site.git ozone-site + fi + + cd ozone-site + + EXISTING_PR=$(gh pr list --repo "$REPO_OWNER/ozone-site" \ + --head "$BRANCH_NAME" --base master --json number --jq '.[0].number' || echo "") + echo "EXISTING_PR=$EXISTING_PR" >> $GITHUB_ENV + + if [ -n "$EXISTING_PR" ]; then + echo "Open PR #$EXISTING_PR exists" + elif [[ -n "$BRANCH_EXISTS" ]]; then + echo "No open PR, but branch exists, resetting to master" + git fetch --depth 1 origin master:master + git reset --hard master + else + echo "No open PR or branch, creating from master" + git checkout -b "$BRANCH_NAME" master fi - name: Check if documentation changed @@ -103,7 +117,7 @@ - name: Checkout ozone repository for script access if: steps.check-changes.outputs.changed == 'true' - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false ref: ${{ github.sha }} @@ -130,27 +144,16 @@ if: steps.check-changes.outputs.changed == 'true' run: | cd ozone-site - - echo "Current branch: $(git branch --show-current)" - echo "Current commit: $(git rev-parse HEAD)" - - # Create/reset branch at current commit - git checkout -B "$BRANCH_NAME" - echo "Created/reset branch $BRANCH_NAME at current commit" - - git add "$TARGET_FILE" - - # Build commit message with JIRA ID if available + COMMIT_MSG="[Auto] Update configuration documentation from ozone $SHA" if [ -n "$JIRA_ID" ]; then COMMIT_MSG="$JIRA_ID. $COMMIT_MSG" fi - + + git add "$TARGET_FILE" git commit -m "$COMMIT_MSG" - - echo "Pushing $BRANCH_NAME to origin" - git push -f origin "$BRANCH_NAME" - + git push --force-with-lease origin "$BRANCH_NAME" + - name: Create or update Pull Request in ozone-site if: steps.check-changes.outputs.changed == 'true' && github.repository == 'apache/ozone' env: @@ -166,10 +169,6 @@ ../ozone-repo/dev-support/ci/pr_body_config_doc.sh \ "$REPO" "$WORKFLOW" "$RUN_ID" "$REF_NAME" "$SHA" "$JIRA_ID" > pr_body.txt - # Check if PR already exists - EXISTING_PR=$(gh pr list --repo "$REPO_OWNER/ozone-site" \ - --head "$BRANCH_NAME" --base master --json number --jq '.[0].number' || echo "") - if [ -n "$EXISTING_PR" ]; then echo "Updating existing PR #$EXISTING_PR with a comment" gh pr comment "$EXISTING_PR" --repo "$REPO_OWNER/ozone-site" --body-file pr_body.txt
diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index bf31f22..f077e6b 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml
@@ -35,16 +35,16 @@ jobs: zizmor: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: security-events: write steps: - name: Checkout project - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@192e21d79ab29983730a13d1382995c2307fbcaa # v0.5.7 + uses: zizmorcore/zizmor-action@6599ee8b7a49aef6a770f63d261d214911a7ce02 # v0.6.0 with: advanced-security: ${{ github.repository_owner == 'apache' }}
diff --git a/AGENTS.md b/AGENTS.md index 5bd695b..85d66a0 100644 --- a/AGENTS.md +++ b/AGENTS.md
@@ -194,7 +194,7 @@ - [`hadoop-ozone/dev-support/checks/README.md`](./hadoop-ozone/dev-support/checks/README.md) - [`hadoop-hdds/dev-support/checkstyle/checkstyle.xml`](./hadoop-hdds/dev-support/checkstyle/checkstyle.xml) - [`dev-support/rat/rat-exclusions.txt`](./dev-support/rat/rat-exclusions.txt) -- [Ozone Enhancement Proposals](https://ozone.apache.org/docs/edge/design/ozone-enhancement-proposals.html) +- [Ozone Enhancement Proposals](https://ozone.apache.org/docs/next/developer-guide/project/enhancement-proposal) ## Security
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ad3fa9a..d705ccc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md
@@ -95,7 +95,7 @@ ## New feature development -For large feature development changes, we use a process called "Ozone Enhancement Proposals" (OEP). This process is designed to ensure that major changes to Ozone are well-designed and have community consensus. If you are planning to propose a significant change, please read the [Ozone Enhancement Proposals](https://ozone.apache.org/docs/edge/design/ozone-enhancement-proposals.html) documentation and create a design document before you start coding. Please note that we only accept design documents in Markdown format; PDF or Google Docs are no longer accepted. +For large feature development changes, we use a process called "Ozone Enhancement Proposals" (OEP). This process is designed to ensure that major changes to Ozone are well-designed and have community consensus. If you are planning to propose a significant change, please read the [Ozone Enhancement Proposals](https://ozone.apache.org/docs/next/developer-guide/project/enhancement-proposal) documentation and create a design document before you start coding. Please note that we only accept design documents in Markdown format; PDF or Google Docs are no longer accepted. ## Contribute your modifications
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java index e7b56a6..824f267 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
@@ -317,6 +317,14 @@ public class OzoneClientConfig { tags = ConfigTag.CLIENT) private boolean enablePutblockPiggybacking = false; + @Config(key = "ozone.client.datastream.putblock.on.close.enabled", + defaultValue = "false", + type = ConfigType.BOOLEAN, + description = "When enabled, use StreamInitWithPutBlock so datanodes commit PutBlock " + + "when the Ratis data stream closes instead of via a separate WriteAsync PutBlock.", + tags = ConfigTag.CLIENT) + private boolean datastreamPutBlockOnCloseEnabled = false; + @Config(key = "ozone.client.key.write.concurrency", defaultValue = "1", description = "Maximum concurrent writes allowed on each key. " + @@ -698,6 +706,14 @@ public void setStreamReadTimeout(Duration streamReadTimeout) { this.streamReadTimeout = streamReadTimeout; } + public boolean isDatastreamPutBlockOnCloseEnabled() { + return datastreamPutBlockOnCloseEnabled; + } + + public void setDatastreamPutBlockOnCloseEnabled(boolean datastreamPutBlockOnCloseEnabled) { + this.datastreamPutBlockOnCloseEnabled = datastreamPutBlockOnCloseEnabled; + } + /** * Enum for indicating what mode to use when combining chunk and block * checksums to define an aggregate FileChecksum. This should be considered
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java index 1ababc7..e3fff75 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
@@ -211,9 +211,12 @@ private DataStreamOutput setupStream(Pipeline pipeline) throws IOException { // TODO: The datanode UUID is not used meaningfully, consider deprecating // it or remove it completely if possible String id = pipeline.getFirstNode().getUuidString(); + ContainerProtos.Type streamInitType = config.isDatastreamPutBlockOnCloseEnabled() + ? ContainerProtos.Type.StreamInitWithPutBlock + : ContainerProtos.Type.StreamInit; ContainerProtos.ContainerCommandRequestProto.Builder builder = ContainerProtos.ContainerCommandRequestProto.newBuilder() - .setCmdType(ContainerProtos.Type.StreamInit) + .setCmdType(streamInitType) .setContainerID(blockID.get().getContainerID()) .setDatanodeUuid(id).setWriteChunk(writeChunkRequest); @@ -416,6 +419,10 @@ public void executePutBlock(boolean close, byteBufferList = null; } waitFuturesComplete(); + if (close && config.isDatastreamPutBlockOnCloseEnabled()) { + // Wait for boundary PutBlock(s) before appending the stream-close PutBlock. + waitPutBlockFuturesComplete(); + } final BlockData blockData = containerBlockData.build(); if (close) { // HDDS-12007 changed datanodes to ignore the following PutBlock request. @@ -437,8 +444,12 @@ public void executePutBlock(boolean close, } } }); + if (config.isDatastreamPutBlockOnCloseEnabled()) { + // PutBlock is supposed to be committed after the data stream close so there + // is no need to continue. + return; + } } - try { XceiverClientReply asyncReply = putBlockAsync(xceiverClient, blockData, close, tokenString); @@ -545,6 +556,19 @@ public void waitFuturesComplete() throws IOException { } } + private void waitPutBlockFuturesComplete() throws IOException { + if (putBlockFutures.isEmpty()) { + return; + } + try { + CompletableFuture.allOf(putBlockFutures.toArray(EMPTY_FUTURE_ARRAY)).get(); + checkOpen(); + } catch (Exception e) { + LOG.warn("Failed to commit PutBlock before stream close: " + e); + throw new IOException(e); + } + } + /** * @param close whether the flush is happening as part of closing the stream */
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/MultipartInputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/MultipartInputStream.java index 221a48b..075ab08 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/MultipartInputStream.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/MultipartInputStream.java
@@ -196,6 +196,10 @@ public boolean readFully(long position, ByteBuffer buffer) throws IOException { final long oldPos = getPos(); seek(position); try { + int remainingBeforeRead = buffer.remaining(); + if (remainingBeforeRead == 0) { + return true; + } read(new ByteBufferReader(buffer) { @Override int readImpl(InputStream inputStream) throws IOException { @@ -203,6 +207,10 @@ int readImpl(InputStream inputStream) throws IOException { .readFully(getBuffer(), false); } }); + if (remainingBeforeRead - buffer.remaining() == 0) { + throw new EOFException("EOF encountered at pos: " + position + + " for key: " + key); + } } finally { seek(oldPos); }
diff --git a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/TestOzoneClientConfig.java b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/TestOzoneClientConfig.java index 5c1eecc..cb05d31 100644 --- a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/TestOzoneClientConfig.java +++ b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/TestOzoneClientConfig.java
@@ -91,4 +91,21 @@ public void testStreamReadConfigParsing() { assertEquals(2 << 20, clientConfig.getStreamReadResponseDataSize()); assertEquals(Duration.ofSeconds(5), clientConfig.getStreamReadTimeout()); } + + @Test + void testDatastreamPutBlockOnCloseEnabledDefault() { + OzoneClientConfig subject = new OzoneConfiguration() + .getObject(OzoneClientConfig.class); + assertFalse(subject.isDatastreamPutBlockOnCloseEnabled()); + } + + @Test + void testDatastreamPutBlockOnCloseConfigParsing() { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.setBoolean("ozone.client.datastream.putblock.on.close.enabled", true); + + OzoneClientConfig subject = conf.getObject(OzoneClientConfig.class); + + assertTrue(subject.isDatastreamPutBlockOnCloseEnabled()); + } }
diff --git a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/MockDatanodePipeline.java b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/MockDatanodePipeline.java index 85341fd..c1c11b0 100644 --- a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/MockDatanodePipeline.java +++ b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/MockDatanodePipeline.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.PutBlockResponseProto; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type; +import org.apache.hadoop.hdds.ratis.ContainerCommandRequestMessage; import org.apache.hadoop.hdds.scm.XceiverClientFactory; import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.XceiverClientRatis; @@ -56,6 +57,7 @@ import org.apache.ratis.io.WriteOption; import org.apache.ratis.protocol.DataStreamReply; import org.apache.ratis.protocol.RoutingTable; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; /** * A stateful test harness that simulates a datanode pipeline for {@link BlockDataStreamOutput} unit tests. @@ -76,6 +78,7 @@ public class MockDatanodePipeline { private final List<byte[]> receivedChunks = Collections.synchronizedList(new ArrayList<>()); private final List<ContainerCommandRequestProto> receivedPutBlocks = Collections.synchronizedList(new ArrayList<>()); private final AtomicInteger watchForCommitCount = new AtomicInteger(0); + private volatile Type streamInitType; // Commit tracking private final AtomicLong nextLogIndex = new AtomicLong(1); @@ -120,8 +123,14 @@ public MockDatanodePipeline(BlockID blockID) throws IOException { // Both overloads must be stubbed: stream(ByteBuffer) and stream(ByteBuffer, RoutingTable) — the pipeline-mode // default is true, so the 2-arg overload is what BlockDataStreamOutput.setupStream calls. DataStreamApi dataStreamApi = mock(DataStreamApi.class); - doReturn(mockDataStreamOutput).when(dataStreamApi).stream(any(ByteBuffer.class)); - doReturn(mockDataStreamOutput).when(dataStreamApi).stream(any(ByteBuffer.class), any(RoutingTable.class)); + doAnswer(invocation -> { + captureStreamInitType(invocation.getArgument(0)); + return mockDataStreamOutput; + }).when(dataStreamApi).stream(any(ByteBuffer.class)); + doAnswer(invocation -> { + captureStreamInitType(invocation.getArgument(0)); + return mockDataStreamOutput; + }).when(dataStreamApi).stream(any(ByteBuffer.class), any(RoutingTable.class)); doReturn(dataStreamApi).when(xceiverClient).getDataStreamApi(); // Setup sendCommandAsync (putBlock) behavior @@ -229,6 +238,10 @@ public int getWatchForCommitCount() { return watchForCommitCount.get(); } + public Type getStreamInitType() { + return streamInitType; + } + /** Concatenate all received chunks into a single byte array. */ public byte[] getAllReceivedData() { int total = receivedChunks.stream().mapToInt(c -> c.length).sum(); @@ -263,6 +276,18 @@ public MockDatanodePipeline failWatchAfter(int n, Supplier<Throwable> err) { // --- Helpers --- + private void captureStreamInitType(ByteBuffer buffer) { + ByteBuffer dup = buffer.duplicate(); + byte[] bytes = new byte[dup.remaining()]; + dup.get(bytes); + try { + streamInitType = ContainerCommandRequestMessage.toProto( + ByteString.copyFrom(bytes), null).getCmdType(); + } catch (Exception e) { + throw new IllegalStateException("Failed to decode stream init request", e); + } + } + private static ContainerCommandResponseProto buildPutBlockResponse(BlockID blockID) { return ContainerCommandResponseProto.newBuilder() .setCmdType(Type.PutBlock)
diff --git a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockDataStreamOutput.java b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockDataStreamOutput.java index 7634328..8f51e3c 100644 --- a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockDataStreamOutput.java +++ b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockDataStreamOutput.java
@@ -30,8 +30,11 @@ import java.util.concurrent.CompletionException; import org.apache.commons.lang3.RandomUtils; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type; import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; /** * Unit tests for {@link BlockDataStreamOutput} exercised through the {@link ByteBufferStreamOutput} interface with a @@ -59,16 +62,33 @@ private static OzoneClientConfig createConfig() { } private BlockDataStreamOutput createStream(MockDatanodePipeline pipeline) throws IOException { + return createStream(pipeline, createConfig()); + } + + private BlockDataStreamOutput createStream( + MockDatanodePipeline pipeline, OzoneClientConfig config) throws IOException { List<StreamBuffer> bufferList = new ArrayList<>(); return new BlockDataStreamOutput( pipeline.getBlockID(), pipeline.getClientFactory(), pipeline.getPipeline(), - createConfig(), + config, null, // no token bufferList); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void streamInitTypeFollowsClientConfig(boolean putBlockOnCloseEnabled) throws Exception { + MockDatanodePipeline pipeline = new MockDatanodePipeline(); + OzoneClientConfig config = createConfig(); + config.setDatastreamPutBlockOnCloseEnabled(putBlockOnCloseEnabled); + try (BlockDataStreamOutput stream = createStream(pipeline, config)) { + Type expected = putBlockOnCloseEnabled ? Type.StreamInitWithPutBlock : Type.StreamInit; + assertEquals(expected, pipeline.getStreamInitType()); + } + } + @Test void writeSubChunkThenClose() throws Exception { MockDatanodePipeline pipeline = new MockDatanodePipeline();
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java index 8e9d4e9..4804bfd 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -39,6 +39,10 @@ public final class HddsConfigKeys { "hdds.heartbeat.recon.initial-interval"; public static final String HDDS_RECON_INITIAL_HEARTBEAT_INTERVAL_DEFAULT = "2s"; + /** Missed heartbeats against one SCM before the DN re-resolves its hostname (HDDS-15533). */ + public static final String HDDS_HEARTBEAT_ADDRESS_REFRESH_MISSED_COUNT_THRESHOLD = + "hdds.heartbeat.address.refresh.missed-count-threshold"; + public static final int HDDS_HEARTBEAT_ADDRESS_REFRESH_MISSED_COUNT_THRESHOLD_DEFAULT = 3; public static final String HDDS_NODE_REPORT_INTERVAL = "hdds.node.report.interval"; public static final String HDDS_NODE_REPORT_INTERVAL_DEFAULT =
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java index 4628905..3e6715b 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
@@ -373,6 +373,7 @@ public static boolean isReadOnly( case PutBlock: case PutSmallFile: case StreamInit: + case StreamInitWithPutBlock: case StreamWrite: case FinalizeBlock: return false;
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java index 12ad150..65acb9d 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java
@@ -206,7 +206,7 @@ public void setIterations(int count) { * Gets the maximum percentage of healthy, in-service datanodes that will be * involved in balancing in one iteration. * - * @return percentage as an integer from 0 up to and including 100 + * @return percentage as an integer greater than 0 up to and including 100 */ public int getMaxDatanodesPercentageToInvolvePerIteration() { return maxDatanodesPercentageToInvolvePerIteration; @@ -254,6 +254,17 @@ public double getMaxDatanodesRatioToInvolvePerIteration() { } /** + * Computes the maximum number of datanodes that may be involved in an + * iteration for the given eligible datanode count. + * + * @param eligibleDatanodeCount number of healthy, in-service datanodes. + * @return maximum datanodes that may be involved in one iteration + */ + public int computeMaxDatanodesToInvolvePerIteration(int eligibleDatanodeCount) { + return (int) (getMaxDatanodesRatioToInvolvePerIteration() * eligibleDatanodeCount); + } + + /** * Sets the maximum percentage of healthy, in-service datanodes that will be * involved in balancing in one iteration. * @@ -266,10 +277,10 @@ public double getMaxDatanodesRatioToInvolvePerIteration() { */ public void setMaxDatanodesPercentageToInvolvePerIteration( int maxDatanodesPercentageToInvolvePerIteration) { - if (maxDatanodesPercentageToInvolvePerIteration < 0 || + if (maxDatanodesPercentageToInvolvePerIteration <= 0 || maxDatanodesPercentageToInvolvePerIteration > 100) { throw new IllegalArgumentException(String.format("Argument %d is " + - "illegal. Percentage must be from 0 up to and including 100.", + "illegal. Percentage must be greater than 0 up to and including 100.", maxDatanodesPercentageToInvolvePerIteration)); } this.maxDatanodesPercentageToInvolvePerIteration =
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/HostAndPort.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/HostAndPort.java index e89ad73..2a142ea 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/HostAndPort.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/HostAndPort.java
@@ -17,7 +17,9 @@ package org.apache.hadoop.hdds.scm.net; +import java.net.InetAddress; import java.net.InetSocketAddress; +import java.util.Objects; import org.apache.hadoop.net.NetUtils; /** @@ -30,14 +32,13 @@ public class HostAndPort { private final String hostAndPortString; private final int hash; /** The address can be updated from time to time. */ - private final InetSocketAddress address; + private volatile InetSocketAddress address; public HostAndPort(String host, int port) { this.host = host; this.port = port; this.hostAndPortString = host + ":" + port; this.hash = host.hashCode() ^ Integer.hashCode(port); - // TODO: HDDS-15533 change the address resolution logic and make this.address threadsafe. this.address = NetUtils.createSocketAddr(hostAndPortString); } @@ -57,6 +58,21 @@ public InetSocketAddress getAddress() { return address; } + /** Re-resolves host:port and returns the new address if its IP changed, else null. No mutation. */ + public InetSocketAddress resolveLatest() { + final InetSocketAddress latest = NetUtils.createSocketAddr(hostAndPortString); + final InetAddress latestIp = latest.getAddress(); + if (latestIp == null || latestIp.equals(address.getAddress())) { + return null; + } + return latest; + } + + /** Commits an address re-resolved via {@link #resolveLatest()}. */ + public void setAddress(InetSocketAddress newAddress) { + this.address = Objects.requireNonNull(newAddress, "newAddress == null"); + } + @Override public int hashCode() { return hash;
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/SimpleStriped.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/SimpleStriped.java index ec83553..390b11e 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/SimpleStriped.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/SimpleStriped.java
@@ -18,7 +18,6 @@ package org.apache.hadoop.hdds.utils; import com.google.common.util.concurrent.Striped; -import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; /** @@ -45,8 +44,7 @@ private SimpleStriped() { * @param fair whether to use a fair ordering policy * @return a new {@code Striped<ReadWriteLock>} */ - public static Striped<ReadWriteLock> readWriteLock(int stripes, - boolean fair) { + public static Striped<ReentrantReadWriteLock> readWriteLock(int stripes, boolean fair) { return Striped.custom(stripes, () -> new ReentrantReadWriteLock(fair)); }
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index bb832d0..3e51a6d 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -4016,6 +4016,16 @@ </property> <property> + <name>hdds.heartbeat.address.refresh.missed-count-threshold</name> + <value>3</value> + <tag>OZONE, DATANODE, HA</tag> + <description>Consecutive heartbeat failures the DataNode tolerates + against one SCM endpoint before re-resolving its hostname. Only + consulted when ozone.client.failover.resolve-needed is true. + </description> + </property> + + <property> <name>ozone.directory.deleting.service.interval</name> <value>1m</value> <tag>OZONE, PERFORMANCE, OM, DELETION</tag>
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/net/TestHostAndPort.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/net/TestHostAndPort.java new file mode 100644 index 0000000..9be015f --- /dev/null +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/net/TestHostAndPort.java
@@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.net; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link HostAndPort} address refresh (HDDS-15533). + */ +public class TestHostAndPort { + + @Test + public void resolveLatestReturnsNullWhenIpUnchanged() { + HostAndPort address = new HostAndPort("127.0.0.1", 9861); + assertNull(address.resolveLatest()); + } + + @Test + public void setAddressRejectsNull() { + HostAndPort address = new HostAndPort("127.0.0.1", 9861); + assertThrows(NullPointerException.class, () -> address.setAddress(null)); + } + + @Test + public void setAddressDoesNotChangeIdentity() throws Exception { + HostAndPort address = new HostAndPort("127.0.0.1", 9861); + InetSocketAddress refreshed = + new InetSocketAddress(InetAddress.getByAddress(new byte[]{10, 0, 0, 7}), 9861); + address.setAddress(refreshed); + assertEquals(refreshed, address.getAddress()); + // equals/hashCode stay keyed on host:port so the instance remains a stable map key. + assertEquals(new HostAndPort("127.0.0.1", 9861), address); + assertEquals(new HostAndPort("127.0.0.1", 9861).hashCode(), address.hashCode()); + } +}
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/TestSimpleStriped.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/TestSimpleStriped.java index ccd80b9..d1ce047 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/TestSimpleStriped.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/TestSimpleStriped.java
@@ -36,8 +36,7 @@ void testReadWriteLocks() { } private void testReadWriteLocks(boolean fair) { - Striped<ReadWriteLock> striped = SimpleStriped.readWriteLock(128, - fair); + Striped<ReentrantReadWriteLock> striped = SimpleStriped.readWriteLock(128, fair); assertEquals(128, striped.size()); ReadWriteLock lock = striped.get("key1"); assertEquals(fair, ((ReentrantReadWriteLock) lock).isFair());
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java index 61d1c49..c5a62d8 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java
@@ -41,6 +41,7 @@ public enum DNAction implements AuditAction { CLOSE_CONTAINER, GET_COMMITTED_BLOCK_LENGTH, STREAM_INIT, + STREAM_INIT_WITH_PUT_BLOCK, FINALIZE_BLOCK, ECHO, GET_CONTAINER_CHECKSUM_INFO,
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java index 47d2983..1564e3d 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
@@ -226,7 +226,8 @@ private ContainerCommandResponseProto dispatchRequest( (cmdType == Type.WriteChunk && dispatcherContext != null && dispatcherContext.getStage() == DispatcherContext.WriteChunkStage.WRITE_DATA) - || (cmdType == Type.StreamInit); + || (cmdType == Type.StreamInit) + || (cmdType == Type.StreamInitWithPutBlock); boolean isWriteCommitStage = (cmdType == Type.WriteChunk && dispatcherContext != null && dispatcherContext.getStage() @@ -925,6 +926,7 @@ private static DNAction getAuditAction(Type cmdType) { case CloseContainer : return DNAction.CLOSE_CONTAINER; case GetCommittedBlockLength : return DNAction.GET_COMMITTED_BLOCK_LENGTH; case StreamInit : return DNAction.STREAM_INIT; + case StreamInitWithPutBlock: return DNAction.STREAM_INIT_WITH_PUT_BLOCK; case FinalizeBlock : return DNAction.FINALIZE_BLOCK; case Echo : return DNAction.ECHO; case GetContainerChecksumInfo: return DNAction.GET_CONTAINER_CHECKSUM_INFO;
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachine.java index b4a9a6e..60bf26a 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachine.java
@@ -152,10 +152,14 @@ public ExecutorService getExecutorService() { */ @Override public void close() { - if (endPoint != null) { - endPoint.close(); + try { + if (endPoint != null) { + endPoint.close(); + } + } finally { + // Always release the executor thread, even if closing the RPC proxy throws. + executorService.shutdown(); } - executorService.shutdown(); } /**
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManager.java index 25be82c..2252d0c 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManager.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManager.java
@@ -22,8 +22,10 @@ import static org.apache.hadoop.hdds.utils.HddsServerUtil.getScmRpcRetryInterval; import static org.apache.hadoop.hdds.utils.HddsServerUtil.getScmRpcTimeOutInMilliseconds; +import com.google.common.annotations.VisibleForTesting; import java.io.Closeable; import java.io.IOException; +import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -139,33 +141,7 @@ public void addSCMServer(HostAndPort address, "Ignoring the request."); return; } - - Configuration hadoopConfig = - LegacyHadoopConfigurationSource.asHadoopConfiguration(this.conf); - RPC.setProtocolEngine( - hadoopConfig, - StorageContainerDatanodeProtocolPB.class, - ProtobufRpcEngine.class); - long version = - RPC.getProtocolVersion(StorageContainerDatanodeProtocolPB.class); - - RetryPolicy retryPolicy = - RetryPolicies.retryUpToMaximumCountWithFixedSleep( - getScmRpcRetryCount(conf), getScmRpcRetryInterval(conf), - TimeUnit.MILLISECONDS); - - StorageContainerDatanodeProtocolPB rpcProxy = RPC.getProtocolProxy( - StorageContainerDatanodeProtocolPB.class, version, - address.getAddress(), UserGroupInformation.getCurrentUser(), hadoopConfig, - NetUtils.getDefaultSocketFactory(hadoopConfig), getRpcTimeout(), - retryPolicy).getProxy(); - - StorageContainerDatanodeProtocolClientSideTranslatorPB rpcClient = - new StorageContainerDatanodeProtocolClientSideTranslatorPB( - rpcProxy); - - EndpointStateMachine endPoint = new EndpointStateMachine(address, - rpcClient, this.conf, threadNamePrefix); + EndpointStateMachine endPoint = buildScmEndpoint(address, address.getAddress(), threadNamePrefix); endPoint.setPassive(false); scmMachines.put(address, endPoint); } finally { @@ -173,6 +149,74 @@ public void addSCMServer(HostAndPort address, } } + @VisibleForTesting + EndpointStateMachine buildScmEndpoint(HostAndPort address, InetSocketAddress dialAddress, + String threadNamePrefix) throws IOException { + Configuration hadoopConfig = + LegacyHadoopConfigurationSource.asHadoopConfiguration(this.conf); + RPC.setProtocolEngine(hadoopConfig, StorageContainerDatanodeProtocolPB.class, + ProtobufRpcEngine.class); + long version = RPC.getProtocolVersion(StorageContainerDatanodeProtocolPB.class); + RetryPolicy retryPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep( + getScmRpcRetryCount(conf), getScmRpcRetryInterval(conf), TimeUnit.MILLISECONDS); + StorageContainerDatanodeProtocolPB rpcProxy = RPC.getProtocolProxy( + StorageContainerDatanodeProtocolPB.class, version, + dialAddress, UserGroupInformation.getCurrentUser(), hadoopConfig, + NetUtils.getDefaultSocketFactory(hadoopConfig), getRpcTimeout(), + retryPolicy).getProxy(); + StorageContainerDatanodeProtocolClientSideTranslatorPB rpcClient = + new StorageContainerDatanodeProtocolClientSideTranslatorPB(rpcProxy); + return new EndpointStateMachine(address, rpcClient, this.conf, threadNamePrefix); + } + + /** + * Re-resolves the active SCM endpoint at {@code address}; on an IP change, rebuilds it under the + * same key and closes the stale proxy. Returns true if rebuilt. + */ + public boolean refreshSCMServer(HostAndPort address, String threadNamePrefix) + throws IOException { + final EndpointStateMachine current; + readLock(); + try { + current = scmMachines.get(address); + if (current == null || current.isPassive()) { + return false; + } + } finally { + readUnlock(); + } + // Resolve outside the lock, but commit the new address only after the replacement is built, + // so a build failure or a lost race never leaves the cached address ahead of the live proxy. + final InetSocketAddress latest = address.resolveLatest(); + if (latest == null) { + return false; + } + final EndpointStateMachine stale; + final InetSocketAddress previous; + writeLock(); + try { + if (scmMachines.get(address) != current) { + return false; + } + EndpointStateMachine rebuilt = buildScmEndpoint(address, latest, threadNamePrefix); + rebuilt.setPassive(false); + previous = address.getAddress(); + address.setAddress(latest); + scmMachines.put(address, rebuilt); + stale = current; + } finally { + writeUnlock(); + } + // The swap is committed; failing to close the stale proxy is cleanup-only, not a refresh failure. + try { + stale.close(); + } catch (RuntimeException e) { + LOG.warn("Failed to close stale endpoint for {}", address, e); + } + LOG.info("SCM endpoint {} re-resolved: {} -> {}", address.getHostAndPortString(), previous, latest); + return true; + } + /** * Adds a new Recon server to the set of endpoints. *
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java index 7cb2455..6c03d14 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
@@ -19,8 +19,12 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_ACTION_MAX_LIMIT; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_ACTION_MAX_LIMIT_DEFAULT; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_ADDRESS_REFRESH_MISSED_COUNT_THRESHOLD; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_ADDRESS_REFRESH_MISSED_COUNT_THRESHOLD_DEFAULT; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_ACTION_MAX_LIMIT; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_ACTION_MAX_LIMIT_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_FAILOVER_RESOLVE_NEEDED_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_FAILOVER_RESOLVE_NEEDED_KEY; import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toLayoutVersionProto; import com.google.common.base.Preconditions; @@ -44,6 +48,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMHeartbeatResponseProto; import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; +import org.apache.hadoop.hdds.utils.ConnectionFailureUtils; import org.apache.hadoop.hdfs.util.EnumCounters; import org.apache.hadoop.ozone.container.common.helpers.DeletedContainerBlocksSummary; import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine; @@ -77,6 +82,8 @@ public class HeartbeatEndpointTask private int maxContainerActionsPerHB; private int maxPipelineActionsPerHB; private HDDSLayoutVersionManager layoutVersionManager; + private final boolean resolveOnFailureEnabled; + private final int refreshThreshold; /** * Constructs a SCM heart beat. @@ -100,6 +107,10 @@ public HeartbeatEndpointTask(EndpointStateMachine rpcEndpoint, } else { this.layoutVersionManager = context.getParent().getLayoutVersionManager(); } + this.resolveOnFailureEnabled = conf.getBoolean(OZONE_CLIENT_FAILOVER_RESOLVE_NEEDED_KEY, + OZONE_CLIENT_FAILOVER_RESOLVE_NEEDED_DEFAULT); + this.refreshThreshold = Math.max(1, conf.getInt(HDDS_HEARTBEAT_ADDRESS_REFRESH_MISSED_COUNT_THRESHOLD, + HDDS_HEARTBEAT_ADDRESS_REFRESH_MISSED_COUNT_THRESHOLD_DEFAULT)); } /** @@ -157,12 +168,33 @@ public EndpointStateMachine.EndPointStates call() throws Exception { // put back the reports which failed to be sent putBackIncrementalReports(requestBuilder); rpcEndpoint.logIfNeeded(ex); + maybeRefreshScmAddress(ex); } finally { rpcEndpoint.unlock(); } return rpcEndpoint.getState(); } + /** + * On a connection-class heartbeat failure past the threshold (and when resolve-needed is on), + * asks the connection manager to re-resolve this SCM peer and rebuild the endpoint. + */ + private void maybeRefreshScmAddress(IOException heartbeatFailure) { + if (!resolveOnFailureEnabled + || rpcEndpoint.isPassive() + || rpcEndpoint.getMissedCount() < refreshThreshold + || !ConnectionFailureUtils.isConnectionFailure(heartbeatFailure)) { + return; + } + try { + context.getParent().getConnectionManager() + .refreshSCMServer(rpcEndpoint.getAddress(), context.getThreadNamePrefix()); + } catch (IOException ex) { + LOG.warn("Failed to refresh SCM address {} after {} missed heartbeats", + rpcEndpoint.getAddress(), rpcEndpoint.getMissedCount(), ex); + } + } + // TODO: Make it generic. private void putBackIncrementalReports( SCMHeartbeatRequestProto.Builder requestBuilder) {
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java index a2ef987..a664ae2 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -364,6 +364,7 @@ static ContainerCommandResponseProto dispatchRequest(KeyValueHandler handler, case WriteChunk: return handler.handleWriteChunk(request, kvContainer, dispatcherContext); case StreamInit: + case StreamInitWithPutBlock: return handler.handleStreamInit(request, kvContainer, dispatcherContext); case ListChunk: return handler.handleUnsupportedOp(request);
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java index 62dbcbe..4692963 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
@@ -209,6 +209,20 @@ public long persistPutBlock(KeyValueContainer container, // container to determine whether the blockCount is already incremented // for this block in the DB or not. long localID = data.getLocalID(); + // For the PutBlock that is endOfBlock and meanwhile bscId = 0, it means + // this PutBlock comes from data stream close without going through the + // Raft, thus there is no log index. In this case, we should not let + // 0 to overwrite previous possible PutBlocks from Ratis log that were + // generated during immediate flushes from the active data stream. Instead, + // we should load the latest bscid and reuse that id. + if (endOfBlock && bcsId == 0) { + BlockData existing = db.getStore().getBlockDataTable() + .get(containerData.getBlockKey(localID)); + if (existing != null) { + bcsId = existing.getBlockCommitSequenceId(); + data.setBlockCommitSequenceId(bcsId); + } + } boolean isBlockInCache = container.isBlockInPendingPutBlockCache(localID); boolean incrBlockCount = false;
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeTable.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeTable.java index ed7a050..fd73f65 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeTable.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeTable.java
@@ -29,11 +29,13 @@ /** * Wrapper class to represent a table in a datanode RocksDB instance. * This class can wrap any existing {@link Table} instance, but will throw - * {@link UnsupportedOperationException} for {@link Table#iterator}. + * {@link UnsupportedOperationException} for {@link Table#iterator} and + * {@link Table#clear()}. * This is because differing schema versions used in datanode DB layouts may * have differing underlying table structures, so iterating a table instance * directly, without taking into account key prefixes, may yield unexpected - * results. + * results, while clearing it may delete data belonging to other logical + * tables or containers. */ public class DatanodeTable<KEY, VALUE> implements Table<KEY, VALUE> { @@ -69,6 +71,12 @@ public void deleteRange(KEY beginKey, KEY endKey) throws RocksDatabaseException, } @Override + public void clear() { + throw new UnsupportedOperationException("Clearing tables directly is not supported for datanode containers due to " + + "differing schema versions."); + } + + @Override public void deleteWithBatch(BatchOperation batch, KEY key) throws CodecException { table.deleteWithBatch(batch, key); }
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java index a8723fc..eaba6d9 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java
@@ -136,29 +136,30 @@ public void cleanup() { } /** - * Because all tables in schema version one map back to the default table, - * directly iterating any of the table instances should be forbidden. - * Otherwise, the iterators for each table would read the entire default - * table, return all database contents, and yield unexpected results. + * Because datanode schemas may map multiple logical tables to the same + * underlying table, directly iterating or clearing any of the table + * instances should be forbidden. Otherwise, iteration may read unrelated + * data and clearing may delete it. * * @throws Exception */ @ParameterizedTest @MethodSource("schemaVersion") - public void testDirectTableIterationDisabled(String schemaVersion) + public void testDirectTableOperationsDisabled(String schemaVersion) throws Exception { setup(schemaVersion); try (DBHandle refCountedDB = BlockUtils.getDB(newKvData(), conf)) { DatanodeStore store = refCountedDB.getStore(); - assertTableIteratorUnsupported(store.getMetadataTable()); - assertTableIteratorUnsupported(store.getBlockDataTable()); - assertTableIteratorUnsupported(store.getDeletedBlocksTable()); + assertTableOperationsUnsupported(store.getMetadataTable()); + assertTableOperationsUnsupported(store.getBlockDataTable()); + assertTableOperationsUnsupported(store.getDeletedBlocksTable()); } } - private void assertTableIteratorUnsupported(Table<?, ?> table) { + private void assertTableOperationsUnsupported(Table<?, ?> table) { assertThrows(UnsupportedOperationException.class, table::iterator); + assertThrows(UnsupportedOperationException.class, table::clear); } /**
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestSCMConnectionManager.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestSCMConnectionManager.java index 0225b6a..ba86a52 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestSCMConnectionManager.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestSCMConnectionManager.java
@@ -18,7 +18,14 @@ package org.apache.hadoop.ozone.container.common.statemachine; import static org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine.EndPointStates.HEARTBEAT; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.net.HostAndPort; import org.junit.jupiter.api.Assertions; @@ -29,6 +36,8 @@ */ public class TestSCMConnectionManager { + private static final InetSocketAddress NEW_IP = newIp(); + @Test public void testRemoveSCMServerDoesNotMarkEndpointShutdown() throws Exception { @@ -46,4 +55,86 @@ public void testRemoveSCMServerDoesNotMarkEndpointShutdown() Assertions.assertEquals(HEARTBEAT, endpoint.getState()); } } + + @Test + public void refreshRebuildsEndpointWhenIpChanges() throws Exception { + try (SCMConnectionManager cm = + new SCMConnectionManager(new OzoneConfiguration())) { + final HostAndPort address = spy(new HostAndPort("127.0.0.1", 9861)); + cm.addSCMServer(address, ""); + final EndpointStateMachine original = cm.getValues().iterator().next(); + doReturn(NEW_IP).when(address).resolveLatest(); + + Assertions.assertTrue(cm.refreshSCMServer(address, "")); + Assertions.assertNotSame(original, cm.getValues().iterator().next()); + Assertions.assertEquals(NEW_IP, address.getAddress()); + } + } + + @Test + public void refreshBuildFailureLeavesEndpointAndAddressUnchanged() + throws Exception { + try (FailingConnectionManager cm = + new FailingConnectionManager(new OzoneConfiguration())) { + final HostAndPort address = spy(new HostAndPort("127.0.0.1", 9861)); + cm.addSCMServer(address, ""); + final EndpointStateMachine original = cm.getValues().iterator().next(); + final InetSocketAddress before = address.getAddress(); + doReturn(NEW_IP).when(address).resolveLatest(); + cm.failBuild = true; + + // Build fails after DNS returned a new IP: the live endpoint and the cached address must both + // stay unchanged, otherwise the DN dials the stale proxy forever while getAddress() reports the + // new IP -- the "stuck until restart" state this feature removes. + Assertions.assertThrows(IOException.class, () -> cm.refreshSCMServer(address, "")); + Assertions.assertSame(original, cm.getValues().iterator().next()); + Assertions.assertEquals(before, address.getAddress()); + } + } + + @Test + public void refreshAbandonedWhenEndpointRemovedDuringResolve() throws Exception { + try (SCMConnectionManager cm = + new SCMConnectionManager(new OzoneConfiguration())) { + final HostAndPort address = spy(new HostAndPort("127.0.0.1", 9861)); + cm.addSCMServer(address, ""); + final InetSocketAddress before = address.getAddress(); + // resolveLatest runs in the unlocked window between the endpoint snapshot and the write lock; + // removing the endpoint right there exercises the lost-race guard deterministically. + doAnswer(inv -> { + cm.removeSCMServer(address); + return NEW_IP; + }).when(address).resolveLatest(); + + Assertions.assertFalse(cm.refreshSCMServer(address, "")); + Assertions.assertTrue(cm.getValues().isEmpty()); + Assertions.assertEquals(before, address.getAddress()); + } + } + + private static InetSocketAddress newIp() { + try { + return new InetSocketAddress(InetAddress.getByAddress(new byte[]{10, 0, 0, 7}), 9861); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + /** SCMConnectionManager whose endpoint build can be forced to fail, to exercise rollback. */ + private static final class FailingConnectionManager extends SCMConnectionManager { + private boolean failBuild; + + FailingConnectionManager(ConfigurationSource conf) { + super(conf); + } + + @Override + EndpointStateMachine buildScmEndpoint(HostAndPort address, InetSocketAddress dialAddress, + String threadNamePrefix) throws IOException { + if (failBuild) { + throw new IOException("simulated build failure"); + } + return super.buildScmEndpoint(address, dialAddress, threadNamePrefix); + } + } }
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/TestHeartbeatEndpointTaskDnsRefresh.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/TestHeartbeatEndpointTaskDnsRefresh.java new file mode 100644 index 0000000..6d9a440 --- /dev/null +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/TestHeartbeatEndpointTaskDnsRefresh.java
@@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.container.common.states.endpoint; + +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_ADDRESS_REFRESH_MISSED_COUNT_THRESHOLD; +import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_FAILOVER_RESOLVE_NEEDED_KEY; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.net.ConnectException; +import java.util.UUID; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto; +import org.apache.hadoop.hdds.scm.net.HostAndPort; +import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; +import org.apache.hadoop.hdfs.util.EnumCounters; +import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; +import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.DatanodeStates; +import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine; +import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager; +import org.apache.hadoop.ozone.container.common.statemachine.StateContext; +import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolClientSideTranslatorPB; +import org.junit.jupiter.api.Test; + +/** + * Verifies a connection-class heartbeat failure past the threshold triggers a DNS re-resolution of + * the SCM peer (HDDS-15533); flag-off, application errors, and below-threshold do not. + */ +public class TestHeartbeatEndpointTaskDnsRefresh { + + private static final HostAndPort SCM = new HostAndPort("test-scm-1", 9861); + + @Test + public void connectionFailureAtThresholdTriggersRefresh() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.setBoolean(OZONE_CLIENT_FAILOVER_RESOLVE_NEEDED_KEY, true); + conf.setInt(HDDS_HEARTBEAT_ADDRESS_REFRESH_MISSED_COUNT_THRESHOLD, 2); + SCMConnectionManager cm = runHeartbeat(conf, 3, new ConnectException("refused")); + verify(cm, times(1)).refreshSCMServer(eq(SCM), any()); + } + + @Test + public void flagOffSuppressesRefresh() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.setBoolean(OZONE_CLIENT_FAILOVER_RESOLVE_NEEDED_KEY, false); + SCMConnectionManager cm = runHeartbeat(conf, 5, new ConnectException("refused")); + verify(cm, never()).refreshSCMServer(any(), any()); + } + + @Test + public void applicationErrorDoesNotTriggerRefresh() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.setBoolean(OZONE_CLIENT_FAILOVER_RESOLVE_NEEDED_KEY, true); + SCMConnectionManager cm = runHeartbeat(conf, 5, new IOException("application-level")); + verify(cm, never()).refreshSCMServer(any(), any()); + } + + @Test + public void belowThresholdDoesNotTriggerRefresh() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.setBoolean(OZONE_CLIENT_FAILOVER_RESOLVE_NEEDED_KEY, true); + conf.setInt(HDDS_HEARTBEAT_ADDRESS_REFRESH_MISSED_COUNT_THRESHOLD, 5); + SCMConnectionManager cm = runHeartbeat(conf, 1, new ConnectException("refused")); + verify(cm, never()).refreshSCMServer(any(), any()); + } + + /** + * Drives one heartbeat that fails with {@code failure}, with the endpoint reporting + * {@code missedCount} missed heartbeats, and returns the mocked connection manager to verify. + */ + private SCMConnectionManager runHeartbeat(OzoneConfiguration conf, long missedCount, + IOException failure) throws Exception { + StorageContainerDatanodeProtocolClientSideTranslatorPB proxy = + mock(StorageContainerDatanodeProtocolClientSideTranslatorPB.class); + when(proxy.sendHeartbeat(any())).thenThrow(failure); + + EndpointStateMachine endpoint = mock(EndpointStateMachine.class); + when(endpoint.getEndPoint()).thenReturn(proxy); + when(endpoint.getAddress()).thenReturn(SCM); + when(endpoint.getMissedCount()).thenReturn(missedCount); + when(endpoint.isPassive()).thenReturn(false); + + SCMConnectionManager connectionManager = mock(SCMConnectionManager.class); + DatanodeStateMachine dsm = mock(DatanodeStateMachine.class); + when(dsm.getConnectionManager()).thenReturn(connectionManager); + when(dsm.getQueuedCommandCount()) + .thenReturn(new EnumCounters<>(SCMCommandProto.Type.class)); + StateContext context = new StateContext(conf, DatanodeStates.RUNNING, dsm, ""); + + HDDSLayoutVersionManager lvm = mock(HDDSLayoutVersionManager.class); + when(lvm.getSoftwareLayoutVersion()).thenReturn(maxLayoutVersion()); + when(lvm.getMetadataLayoutVersion()).thenReturn(maxLayoutVersion()); + + DatanodeDetails dn = DatanodeDetails.newBuilder() + .setUuid(UUID.randomUUID()) + .setHostName("localhost") + .setIpAddress("127.0.0.1") + .build(); + + HeartbeatEndpointTask.newBuilder() + .setConfig(conf) + .setDatanodeDetails(dn) + .setContext(context) + .setLayoutVersionManager(lvm) + .setEndpointStateMachine(endpoint) + .build() + .call(); + return connectionManager; + } +}
diff --git a/hadoop-hdds/docs/content/design/lifecycle-task-resume.md b/hadoop-hdds/docs/content/design/lifecycle-task-resume.md index d800e21..f42308f 100644 --- a/hadoop-hdds/docs/content/design/lifecycle-task-resume.md +++ b/hadoop-hdds/docs/content/design/lifecycle-task-resume.md
@@ -1,23 +1,28 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ +--- +title: Resumable Lifecycle Scans +summary: Persist lifecycle scan pointers so OM leader failover can resume bucket scans +date: 2026-07-13 +jira: HDDS-15447 +status: implemented +author: Sammi Chen +--- +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at -# Design for Resumable Lifecycle Scans(HDDS-8342) + http://www.apache.org/licenses/LICENSE-2.0 -## Problem Statement: + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +# Design for Resumable Lifecycle Scans(HDDS-15447) + +## Problem Statement The `HDDS-8342` branch introduces the `KeyLifecycleService`, a background service running on the Ozone Manager (OM) Leader to enforce bucket lifecycle rules (expiration, moving to trash, and aborting incomplete multipart uploads). The entire bucket is scanned in a single `call()` execution. If the OM restarts, crashes, or a leader transfer occurs, the scan state is lost. The new leader must restart the scan from the beginning.
diff --git a/hadoop-hdds/docs/content/tools/_index.md b/hadoop-hdds/docs/content/tools/_index.md index aa708e2..3a8bf08 100644 --- a/hadoop-hdds/docs/content/tools/_index.md +++ b/hadoop-hdds/docs/content/tools/_index.md
@@ -43,6 +43,8 @@ * **sh** - Primary command line interface for ozone to manage volumes/buckets/keys. * **fs** - Runs a command on ozone file system (similar to `hdfs dfs`) + * **local** - Runs a single-node local Ozone cluster (SCM, OM, and datanodes) + in one process for development. * **version** - Prints the version of Ozone and HDDS.
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmTopologyClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmTopologyClient.java index d595bd6..7abd2c9 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmTopologyClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmTopologyClient.java
@@ -17,7 +17,6 @@ package org.apache.hadoop.hdds.scm.client; -import static java.util.Objects.requireNonNull; import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_NETWORK_TOPOLOGY_REFRESH_DURATION; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_NETWORK_TOPOLOGY_REFRESH_DURATION_DEFAULT; @@ -60,8 +59,7 @@ public ScmTopologyClient( } public NetworkTopology getClusterMap() { - return requireNonNull(cache.get(), - "ScmBlockLocationClient must have been initialized already."); + return cache.get(); } public void start(ConfigurationSource conf) throws IOException {
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java index 39c47ae..7808cb2 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
@@ -1005,9 +1005,9 @@ public StartContainerBalancerResponseProto startContainerBalancer( } if (maxDatanodesPercentageToInvolvePerIteration.isPresent()) { int mdti = maxDatanodesPercentageToInvolvePerIteration.get(); - Preconditions.checkState(mdti >= 0, + Preconditions.checkState(mdti > 0, "Max Datanodes Percentage To Involve Per Iteration must be " + - "greater than equal to zero."); + "greater than zero."); Preconditions.checkState(mdti <= 100, "Max Datanodes Percentage To Involve Per Iteration must be " + "lesser than equal to hundred.");
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java index 400547a..e740e8b 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java
@@ -134,13 +134,35 @@ default VALUE getReadCopy(KEY key) throws RocksDatabaseException, CodecException void deleteWithBatch(BatchOperation batch, KEY key) throws CodecException; /** - * Deletes a range of keys from the metadata store. + * Deletes a range of keys from this table. * - * @param beginKey start metadata key - * @param endKey end metadata key + * @param beginKey start key (inclusive) + * @param endKey end key (exclusive) */ void deleteRange(KEY beginKey, KEY endKey) throws RocksDatabaseException, CodecException; + /** + * Deletes all entries from this table. + * Note: only entries in the underlying DB are deleted; the table cache + * (if any) is not affected. Callers must ensure the cache stays empty + * (or is separately invalidated) for the duration of this operation, + * e.g. by holding exclusive access to the table. + */ + default void clear() throws RocksDatabaseException, CodecException { + final KEY beginKey; + final KEY endKey; + try (TableIterator<KEY, KEY> keyIterator = keyIterator()) { + if (!keyIterator.hasNext()) { + return; + } + beginKey = keyIterator.next(); + keyIterator.seekToLast(); + endKey = keyIterator.next(); + } + deleteRange(beginKey, endKey); + delete(endKey); + } + /** The same as iterator(null, KEY_AND_VALUE). */ default KeyValueIterator<KEY, VALUE> iterator() throws RocksDatabaseException, CodecException { return iterator(null, IteratorType.KEY_AND_VALUE);
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java index 59e9245..7a73b89 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java
@@ -397,6 +397,15 @@ public void deleteRange(KEY beginKey, KEY endKey) throws RocksDatabaseException, rawTable.deleteRange(encodeKey(beginKey), encodeKey(endKey)); } + /** + * Clears the raw table directly: the default implementation would decode and re-encode the boundary keys, + * which can miss persisted keys whose bytes do not round-trip through the codec. + */ + @Override + public void clear() throws RocksDatabaseException, CodecException { + rawTable.clear(); + } + @Override public KeyValueIterator<KEY, VALUE> iterator(KEY prefix, IteratorType type) throws RocksDatabaseException, CodecException {
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/InMemoryTestTable.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/InMemoryTestTable.java index 1dbb502..2f5b8be 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/InMemoryTestTable.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/InMemoryTestTable.java
@@ -96,6 +96,11 @@ public void deleteRange(KEY beginKey, KEY endKey) { } @Override + public void clear() { + map.clear(); + } + + @Override public KeyValueIterator<KEY, VALUE> iterator(KEY prefix, IteratorType type) { throw new UnsupportedOperationException(); }
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestTypedTable.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestTypedTable.java index 6e1ccc5..250d221 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestTypedTable.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestTypedTable.java
@@ -160,6 +160,54 @@ static <T> void assertEqualsSupportingByteArray(T left, T right) { } @Test + public void testClear() throws Exception { + final TypedTable<Long, String> table = newTypedTable(1, LongCodec.get(), StringCodec.get()); + + table.clear(); + assertTrue(table.isEmpty()); + + table.put(1L, "one"); + table.clear(); + assertTrue(table.isEmpty()); + + table.put(1L, "one"); + table.put(2L, "two"); + table.put(3L, "three"); + table.clear(); + assertTrue(table.isEmpty()); + + table.put(4L, "four"); + assertEquals("four", table.get(4L)); + } + + @Test + public void testClearMalformedKey() throws Exception { + final RDBTable rawTable = rdb.getTable(families.get(2)); + final TypedTable<String, String> table = + new TypedTable<>(rawTable, StringCodec.get(), StringCodec.get(), TableCache.CacheType.PARTIAL_CACHE); + + // The last key decodes with replacement characters, so it does not re-encode back to the same bytes; + // see TestCodec#testStringCodecMalformedUtf8String. + final byte[] malformed = {(byte) 0xC3, (byte) '/', 0, 0, 0, 1}; + rawTable.put(malformed, StringCodec.get().toPersistedFormat("value")); + + table.clear(); + + assertTrue(table.isEmpty()); + } + + @Test + public void testClearInMemoryTable() throws Exception { + final Table<Long, String> table = new InMemoryTestTable<>(); + table.put(1L, "one"); + table.put(2L, "two"); + + table.clear(); + + assertTrue(table.isEmpty()); + } + + @Test public void testEmptyStringCodecBuffer() throws Exception { final StringCodec codec = StringCodec.get(); assertTrue(codec.supportCodecBuffer());
diff --git a/hadoop-hdds/rocks-native/dev-support/findbugsExcludeFile.xml b/hadoop-hdds/rocks-native/dev-support/findbugsExcludeFile.xml index 40d78d0..9b97abe 100644 --- a/hadoop-hdds/rocks-native/dev-support/findbugsExcludeFile.xml +++ b/hadoop-hdds/rocks-native/dev-support/findbugsExcludeFile.xml
@@ -15,4 +15,13 @@ limitations under the License. --> <FindBugsFilter> + <!-- MergedKeyValue owns key/value arrays copied once at emit time; callers must not mutate. --> + <Match> + <Class name="org.apache.hadoop.hdds.utils.db.LatestVersionedKWayMergeIterator$MergedKeyValue"/> + <Or> + <Method name="getUserKey"/> + <Method name="getValue"/> + </Or> + <Bug pattern="EI_EXPOSE_REP"/> + </Match> </FindBugsFilter>
diff --git a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/LatestVersionedKWayMergeIterator.java b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/LatestVersionedKWayMergeIterator.java new file mode 100644 index 0000000..fc072c7 --- /dev/null +++ b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/LatestVersionedKWayMergeIterator.java
@@ -0,0 +1,504 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.utils.db; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.primitives.UnsignedLong; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.PriorityQueue; +import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.hdds.utils.db.managed.ManagedOptions; +import org.apache.hadoop.ozone.util.ClosableIterator; + +/** + * K-way merge over RocksDB SST files for snapshot diff. + * <p> + * A single min-heap orders source heads by user key. For each user key, all versions + * are drained and the latest value and latest tombstone are tracked, then snapshot-diff + * emit rules apply: emit the latest tombstone and/or latest value, including both when + * a delete is followed by a newer recreate. + * <p> + * When constructed with an exclusive minimum sequence number {@code S}, each SST + * source skips entries whose sequence is {@code <= S} while advancing. + */ +public final class LatestVersionedKWayMergeIterator implements + ClosableIterator<LatestVersionedKWayMergeIterator.MergedKeyValue> { + + /** RocksDB {@code ValueType::kTypeValue}. */ + public static final int ROCKS_TYPE_VALUE = 1; + private static final int DEFAULT_READ_AHEAD_SIZE = 2 * 1024 * 1024; + + private final ManagedOptions options; + private final List<ClosableIterator<? extends MergeHead>> iterators; + private final Long exclusiveMinSequenceNumber; + + private final PriorityQueue<HeapEntry> sourceHeads; + + private List<MergedKeyValue> emitQueue; + private boolean initialized; + + public static LatestVersionedKWayMergeIterator overRawSstFiles(Collection<Path> sstFiles) { + return overRawSstFiles(sstFiles, DEFAULT_READ_AHEAD_SIZE, null); + } + + public static LatestVersionedKWayMergeIterator overRawSstFiles(Collection<Path> sstFiles, + int readAheadSizePerFile) { + return overRawSstFiles(sstFiles, readAheadSizePerFile, null); + } + + /** + * Opens one iterator per SST file and merges them. + * + * @param exclusiveMinSequenceNumber when non-null, each source skips entries with + * sequence {@code <=} this value while advancing; when null, no entries are skipped + */ + public static LatestVersionedKWayMergeIterator overRawSstFiles(Collection<Path> sstFiles, + int readAheadSizePerFile, Long exclusiveMinSequenceNumber) { + Objects.requireNonNull(sstFiles, "sstFiles cannot be null"); + ManagedOptions options = new ManagedOptions(); + List<ClosableIterator<? extends MergeHead>> sources = new ArrayList<>(sstFiles.size()); + try { + for (Path file : sstFiles) { + sources.add(new RawSstIterator(options, file, readAheadSizePerFile)); + } + return new LatestVersionedKWayMergeIterator(options, sources, exclusiveMinSequenceNumber); + } catch (RuntimeException e) { + IOUtils.closeQuietly(sources); + options.close(); + throw e; + } + } + + public static LatestVersionedKWayMergeIterator overRawSstFilesFromSequence(Collection<Path> sstFiles, + long exclusiveMinSequenceNumber) { + return overRawSstFiles(sstFiles, DEFAULT_READ_AHEAD_SIZE, exclusiveMinSequenceNumber); + } + + @VisibleForTesting + public static LatestVersionedKWayMergeIterator forTest( + List<ClosableIterator<MergedKeyValue>> iterators, Long exclusiveMinSequenceNumber) { + List<ClosableIterator<? extends MergeHead>> sources = new ArrayList<>(iterators.size()); + sources.addAll(iterators); + return new LatestVersionedKWayMergeIterator(null, sources, exclusiveMinSequenceNumber); + } + + private LatestVersionedKWayMergeIterator( + ManagedOptions options, + List<ClosableIterator<? extends MergeHead>> iterators, + Long exclusiveMinSequenceNumber) { + this.options = options; + this.iterators = new ArrayList<>(Objects.requireNonNull(iterators, "iterators cannot be null")); + this.exclusiveMinSequenceNumber = exclusiveMinSequenceNumber; + this.sourceHeads = new PriorityQueue<>(Math.max(this.iterators.size(), 1)); + this.emitQueue = new ArrayList<>(); + } + + @Override + public boolean hasNext() { + if (!emitQueue.isEmpty()) { + return true; + } + return advance(); + } + + @Override + public MergedKeyValue next() { + if (!hasNext()) { + throw new NoSuchElementException("No more elements found."); + } + return emitQueue.remove(0); + } + + private boolean advance() { + if (!initialized) { + initHeap(); + initialized = true; + } + + while (emitQueue.isEmpty() && !sourceHeads.isEmpty()) { + processNextUserKey(); + } + + return !emitQueue.isEmpty(); + } + + private void processNextUserKey() { + if (sourceHeads.isEmpty()) { + return; + } + + byte[] nextKey = sourceHeads.peek().current.getUserKey(); + MergeHead latestValue = null; + long latestValueSeq = -1L; + MergeHead latestTombstone = null; + long latestTombstoneSeq = -1L; + + while (heapHasUserKey(nextKey)) { + List<HeapEntry> polled = new ArrayList<>(); + while (heapHasUserKey(nextKey)) { + HeapEntry entry = sourceHeads.poll(); + MergeHead head = entry.current; + if (head.isTombstone()) { + if (head.getSequence() > latestTombstoneSeq) { + latestTombstone = head; + latestTombstoneSeq = head.getSequence(); + } + } else if (head.getSequence() > latestValueSeq) { + latestValue = head; + latestValueSeq = head.getSequence(); + } + polled.add(entry); + } + for (HeapEntry entry : polled) { + if (entry.current == latestValue && entry.current instanceof RawSstHeapHead) { + ((RawSstHeapHead) entry.current).snapshotValue(); + } + entry.advance(); + if (entry.current != null) { + sourceHeads.offer(entry); + } + } + } + + emitForUserKey(latestValue, latestTombstone); + } + + private void emitForUserKey(MergeHead latestValue, MergeHead latestTombstone) { + if (latestValue != null && latestTombstone != null) { + if (latestValue.getSequence() > latestTombstone.getSequence()) { + emitQueue.add(latestTombstone.toMergedKeyValue()); + emitQueue.add(latestValue.toMergedKeyValue()); + } else { + emitQueue.add(latestTombstone.toMergedKeyValue()); + } + } else if (latestValue != null) { + emitQueue.add(latestValue.toMergedKeyValue()); + } else if (latestTombstone != null) { + emitQueue.add(latestTombstone.toMergedKeyValue()); + } + } + + private boolean heapHasUserKey(byte[] userKey) { + return !sourceHeads.isEmpty() + && compareUserKeys(sourceHeads.peek().current.getUserKey(), userKey) == 0; + } + + private void initHeap() { + for (int idx = 0; idx < iterators.size(); idx++) { + ClosableIterator<? extends MergeHead> iterator = iterators.get(idx); + HeapEntry entry = new HeapEntry(idx, iterator); + if (entry.current != null) { + sourceHeads.offer(entry); + } + } + } + + @Override + public void close() { + IOUtils.closeQuietly(iterators); + if (options != null) { + options.close(); + } + } + + private static int compareUserKeys(byte[] left, byte[] right) { + if (left == right) { + return 0; + } + if (left == null) { + return -1; + } + if (right == null) { + return 1; + } + int minLength = Math.min(left.length, right.length); + for (int i = 0; i < minLength; i++) { + int l = left[i] & 0xff; + int r = right[i] & 0xff; + if (l != r) { + return Integer.compare(l, r); + } + } + return Integer.compare(left.length, right.length); + } + + private static int compareHeapOrder(MergeHead left, int leftIndex, + MergeHead right, int rightIndex) { + int keyCompare = compareUserKeys(left.getUserKey(), right.getUserKey()); + if (keyCompare != 0) { + return keyCompare; + } + int seqCompare = Long.compare(right.getSequence(), left.getSequence()); + if (seqCompare != 0) { + return seqCompare; + } + int tombstoneCompare = Boolean.compare(right.isTombstone(), left.isTombstone()); + if (tombstoneCompare != 0) { + return tombstoneCompare; + } + return Integer.compare(leftIndex, rightIndex); + } + + private interface MergeHead { + byte[] getUserKey(); + + long getSequence(); + + boolean isTombstone(); + + MergedKeyValue toMergedKeyValue(); + } + + private static final class RawSstHeapHead implements MergeHead { + private final byte[] key; + private final long sequence; + private final int type; + private final CodecBuffer valueBuffer; + private byte[] snapshottedValue; + + RawSstHeapHead(byte[] key, long sequence, int type, CodecBuffer valueBuffer) { + this.key = Objects.requireNonNull(key, "key cannot be null"); + this.sequence = sequence; + this.type = type; + this.valueBuffer = valueBuffer; + } + + void snapshotValue() { + if (snapshottedValue != null || isTombstone() || valueBuffer == null) { + return; + } + snapshottedValue = copyBuffer(valueBuffer); + } + + @Override + public byte[] getUserKey() { + return key; + } + + @Override + public long getSequence() { + return sequence; + } + + @Override + public boolean isTombstone() { + return type != ROCKS_TYPE_VALUE; + } + + @Override + public MergedKeyValue toMergedKeyValue() { + byte[] value = snapshottedValue; + if (value == null && !isTombstone() && valueBuffer != null) { + value = copyBuffer(valueBuffer); + } + return new MergedKeyValue(key, UnsignedLong.fromLongBits(sequence), type, value); + } + } + + private final class HeapEntry implements Comparable<HeapEntry> { + private final int index; + private final ClosableIterator<? extends MergeHead> iterator; + private MergeHead current; + + private HeapEntry(int index, ClosableIterator<? extends MergeHead> iterator) { + this.index = index; + this.iterator = iterator; + advance(); + } + + private void advance() { + while (true) { + if (!iterator.hasNext()) { + current = null; + iterator.close(); + return; + } + MergeHead next = iterator.next(); + if (exclusiveMinSequenceNumber == null + || next.getSequence() > exclusiveMinSequenceNumber) { + current = next; + return; + } + } + } + + @Override + public int compareTo(HeapEntry other) { + return compareHeapOrder( + this.current, this.index, other.current, other.index); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof HeapEntry)) { + return false; + } + HeapEntry other = (HeapEntry) o; + return index == other.index; + } + + @Override + public int hashCode() { + return index; + } + } + + private static final class RawSstIterator implements ClosableIterator<RawSstHeapHead> { + private final ManagedRawSSTFileReader reader; + private final ManagedRawSSTFileIterator<ManagedRawSSTFileIterator.KeyValue> iterator; + private boolean closed; + + private RawSstIterator(ManagedOptions options, Path file, int readAheadSize) { + ManagedRawSSTFileReader openedReader = new ManagedRawSSTFileReader( + options, file.toAbsolutePath().toString(), readAheadSize); + ManagedRawSSTFileIterator<ManagedRawSSTFileIterator.KeyValue> openedIterator; + try { + openedIterator = openedReader.newIterator( + kv -> kv, null, null, IteratorType.KEY_AND_VALUE); + } catch (RuntimeException e) { + openedReader.close(); + throw e; + } + this.reader = openedReader; + this.iterator = openedIterator; + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public RawSstHeapHead next() { + ManagedRawSSTFileIterator.KeyValue keyValue = iterator.next(); + int type = keyValue.getType(); + CodecBuffer valueBuffer = type == ROCKS_TYPE_VALUE ? keyValue.getValue() : null; + return new RawSstHeapHead( + copyBuffer(keyValue.getKey()), + keyValue.getSequence().longValue(), + type, + valueBuffer); + } + + @Override + public void close() { + if (closed) { + return; + } + closed = true; + iterator.close(); + reader.close(); + } + } + + /** + * A merged RocksDB internal key after applying snapshot-diff emit rules across + * multiple SST sources. + * <p> + * Each instance represents one emitted version for a user key: either a value + * ({@link #ROCKS_TYPE_VALUE}) or a tombstone (any other RocksDB value type). + * When the latest value has a higher sequence than the latest tombstone for + * the same user key, both are emitted (delete followed by recreate). Otherwise + * only the winning tombstone or value is emitted. + * <p> + * Key and value arrays are owned by this object. Callers must treat returned + * {@code byte[]} references as read-only; snapshot-diff integration should + * consume them once without retaining mutable aliases. + */ + public static final class MergedKeyValue implements MergeHead { + private final byte[] key; + private final UnsignedLong sequence; + private final int type; + private final byte[] value; + + /** Creates a merged entry for unit tests. */ + @VisibleForTesting + public static MergedKeyValue of(byte[] key, long sequence, int type, byte[] value) { + return new MergedKeyValue( + Arrays.copyOf(key, key.length), + UnsignedLong.fromLongBits(sequence), + type, + value == null ? null : Arrays.copyOf(value, value.length)); + } + + private MergedKeyValue(byte[] key, UnsignedLong sequence, int type, byte[] value) { + this.key = key; + this.sequence = Objects.requireNonNull(sequence, "sequence cannot be null"); + this.type = type; + this.value = value; + } + + /** Returns the user key bytes shared by all versions merged for this emit. */ + @Override + public byte[] getUserKey() { + return key; + } + + /** Returns the RocksDB sequence number as a signed {@code long}. */ + @Override + public long getSequence() { + return sequence.longValue(); + } + + /** Returns the RocksDB internal value type for this record. */ + public int getValueType() { + return type; + } + + /** Returns the RocksDB sequence number. */ + public UnsignedLong getSequenceNumber() { + return sequence; + } + + /** Returns the value bytes, or {@code null} for tombstones. */ + public byte[] getValue() { + return value; + } + + /** Returns {@code true} when this record is a tombstone rather than a value. */ + @Override + public boolean isTombstone() { + return type != ROCKS_TYPE_VALUE; + } + + @Override + public MergedKeyValue toMergedKeyValue() { + return this; + } + } + + private static byte[] copyBuffer(CodecBuffer buffer) { + if (buffer == null) { + return null; + } + ByteBuffer byteBuffer = buffer.asReadOnlyByteBuffer(); + byte[] bytes = new byte[byteBuffer.remaining()]; + byteBuffer.get(bytes); + return bytes; + } +}
diff --git a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/TestLatestVersionedKWayMergeIterator.java b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/TestLatestVersionedKWayMergeIterator.java new file mode 100644 index 0000000..76c7936 --- /dev/null +++ b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/TestLatestVersionedKWayMergeIterator.java
@@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.utils.db; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.hadoop.hdds.utils.db.LatestVersionedKWayMergeIterator.MergedKeyValue; +import org.apache.hadoop.ozone.util.ClosableIterator; +import org.junit.jupiter.api.Named; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class TestLatestVersionedKWayMergeIterator { + + private static Stream<Arguments> mergeScenarios() { + return Stream.of( + Named.of("recreate: tombstone then newer value emits both", + scenario( + expected(kv("k1", 2, 0, null), kv("k1", 3, 1, "v3")), + source(kv("k1", 1, 1, "v1"), kv("k1", 2, 0, null)), + source(kv("k1", 3, 1, "v3")))), + Named.of("delete only: latest tombstone wins", + scenario( + expected(kv("k1", 5, 0, null)), + source(kv("k1", 1, 1, "v1")), + source(kv("k1", 5, 0, null)))), + Named.of("value only: latest value wins", + scenario( + expected(kv("k1", 5, 1, "v5")), + source(kv("k1", 1, 1, "v1")), + source(kv("k1", 5, 1, "v5")))), + Named.of("three-file worked example", + scenario( + expected(kv("k1", 30, 1, "v30"), kv("k2", 15, 0, null), kv("k2", 25, 1, "v25")), + source(kv("k1", 10, 1, "v10"), kv("k1", 5, 1, "v5"), kv("k2", 20, 1, "v20")), + source(kv("k1", 3, 1, "v3"), kv("k1", 15, 1, "v15"), kv("k2", 15, 0, null)), + source(kv("k1", 1, 1, "v1"), kv("k1", 30, 1, "v30"), kv("k2", 25, 1, "v25")))), + Named.of("multi-key: recreate on k1, delete-only on k2", + scenario( + expected(kv("k1", 3, 0, null), kv("k1", 10, 1, "v10"), kv("k2", 15, 0, null)), + source(kv("k1", 10, 1, "v10"), kv("k1", 3, 0, null), kv("k2", 10, 1, "v10")), + source(kv("k2", 15, 0, null)))), + Named.of("duplicate tombstones deduped to highest sequence", + scenario( + expected(kv("k1", 7, 0, null)), + source(kv("k1", 4, 0, null), kv("k1", 2, 0, null)), + source(kv("k1", 7, 0, null)))), + Named.of("multiple recreate cycles on same key", + scenario( + expected(kv("k1", 4, 0, null), kv("k1", 6, 1, "v6")), + source( + kv("k1", 1, 1, "v1"), + kv("k1", 2, 0, null), + kv("k1", 3, 1, "v3"), + kv("k1", 4, 0, null), + kv("k1", 5, 1, "v5"), + kv("k1", 6, 1, "v6")))), + Named.of("interleaved keys preserve user-key order", + scenario( + expected(kv("a", 1, 1, "a1"), kv("b", 2, 1, "b1"), kv("c", 3, 1, "c1")), + source(kv("a", 1, 1, "a1"), kv("c", 3, 1, "c1")), + source(kv("b", 2, 1, "b1")))), + Named.of("empty source files are ignored", + scenario( + expected(kv("k1", 2, 1, "v2")), + source(kv("k1", 1, 1, "v1")), + source(), + source(kv("k1", 2, 1, "v2")))) + ).map(Arguments::of); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("mergeScenarios") + void testMergeMatchesExpectedOutput(MergeScenario scenario) { + assertResultsEqual(scenario.expected, merge(scenario.sources)); + } + + @Test + void testExclusiveMinSequenceNumberFiltersPerKey() { + long exclusiveMinSequenceNumber = 5L; + List<List<MergedKeyValue>> sources = Arrays.asList( + Arrays.asList( + kv("k1", 7, 1, "v7"), + kv("k2", 1, 1, "v1"), + kv("k3", 6, 0, null), + kv("k4", 10, 1, "v10")), + Arrays.asList( + kv("k1", 3, 0, null), + kv("k2", 3, 0, null), + kv("k3", 8, 1, "v8"))); + List<MergedKeyValue> expected = Arrays.asList( + kv("k1", 7, 1, "v7"), + kv("k3", 6, 0, null), + kv("k3", 8, 1, "v8"), + kv("k4", 10, 1, "v10")); + + assertResultsEqual(expected, merge(sources, exclusiveMinSequenceNumber)); + } + + private static List<MergedKeyValue> merge(List<List<MergedKeyValue>> sources) { + return merge(sources, null); + } + + private static List<MergedKeyValue> merge(List<List<MergedKeyValue>> sources, + Long exclusiveMinSequenceNumber) { + List<ClosableIterator<MergedKeyValue>> iterators = sources.stream() + .map(ListIterator::new) + .collect(Collectors.toList()); + + List<MergedKeyValue> results = new ArrayList<>(); + try (LatestVersionedKWayMergeIterator iterator = + LatestVersionedKWayMergeIterator.forTest(iterators, exclusiveMinSequenceNumber)) { + while (iterator.hasNext()) { + results.add(iterator.next()); + } + } + return results; + } + + private static void assertResultsEqual(List<MergedKeyValue> expected, List<MergedKeyValue> actual) { + assertEquals(expected.size(), actual.size(), + () -> "expected=" + describe(expected) + " actual=" + describe(actual)); + + for (int i = 0; i < expected.size(); i++) { + MergedKeyValue exp = expected.get(i); + MergedKeyValue act = actual.get(i); + assertArrayEquals(exp.getUserKey(), act.getUserKey(), "key mismatch at index " + i); + assertEquals(exp.getSequence(), act.getSequence(), "sequence mismatch at index " + i); + assertEquals(exp.getValueType(), act.getValueType(), "type mismatch at index " + i); + if (exp.getValue() == null) { + assertNull(act.getValue(), "value should be null at index " + i); + } else { + assertArrayEquals(exp.getValue(), act.getValue(), "value mismatch at index " + i); + } + } + } + + private static String describe(List<MergedKeyValue> entries) { + StringBuilder sb = new StringBuilder("["); + for (MergedKeyValue entry : entries) { + sb.append('{') + .append(asString(entry.getUserKey())) + .append(", seq=").append(entry.getSequence()) + .append(", type=").append(entry.getValueType()) + .append("} "); + } + return sb.append(']').toString(); + } + + private static MergeScenario scenario(Expected expected, Source... sources) { + List<List<MergedKeyValue>> sourceKvs = new ArrayList<>(); + for (Source source : sources) { + sourceKvs.add(source.entries); + } + return new MergeScenario(sourceKvs, expected.entries); + } + + private static Expected expected(MergedKeyValue... entries) { + return new Expected(Arrays.asList(entries)); + } + + private static Source source(MergedKeyValue... entries) { + return new Source(Arrays.asList(entries)); + } + + private static MergedKeyValue kv(String key, long sequence, int type, String value) { + return MergedKeyValue.of( + key.getBytes(StandardCharsets.UTF_8), + sequence, + type, + value == null ? null : value.getBytes(StandardCharsets.UTF_8)); + } + + private static String asString(byte[] bytes) { + return new String(bytes, StandardCharsets.UTF_8); + } + + private static final class MergeScenario { + private final List<List<MergedKeyValue>> sources; + private final List<MergedKeyValue> expected; + + private MergeScenario(List<List<MergedKeyValue>> sources, + List<MergedKeyValue> expected) { + this.sources = sources; + this.expected = expected; + } + } + + private static final class Expected { + private final List<MergedKeyValue> entries; + + private Expected(List<MergedKeyValue> entries) { + this.entries = entries; + } + } + + private static final class Source { + private final List<MergedKeyValue> entries; + + private Source(List<MergedKeyValue> entries) { + this.entries = entries; + } + } + + private static final class ListIterator implements ClosableIterator<MergedKeyValue> { + private final Iterator<MergedKeyValue> iterator; + + private ListIterator(List<MergedKeyValue> entries) { + this.iterator = entries.iterator(); + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public MergedKeyValue next() { + return iterator.next(); + } + + @Override + public void close() { + // Nothing to close. + } + } +}
diff --git a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/TestLatestVersionedKWayMergeIteratorOverSst.java b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/TestLatestVersionedKWayMergeIteratorOverSst.java new file mode 100644 index 0000000..3b7c9e4 --- /dev/null +++ b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/TestLatestVersionedKWayMergeIteratorOverSst.java
@@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.utils.db; + +import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_PROPERTY; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.hadoop.hdds.utils.NativeLibraryNotLoadedException; +import org.apache.hadoop.hdds.utils.db.LatestVersionedKWayMergeIterator.MergedKeyValue; +import org.apache.hadoop.hdds.utils.db.TestRawSstFileRecords.SourceRecord; +import org.apache.hadoop.hdds.utils.db.managed.ManagedColumnFamilyOptions; +import org.apache.hadoop.hdds.utils.db.managed.ManagedDBOptions; +import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.junit.jupiter.api.io.TempDir; +import org.rocksdb.ColumnFamilyDescriptor; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.FlushOptions; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; + +/** + * End-to-end test over real SST files produced by RocksDB flush. + * <p> + * {@link org.apache.hadoop.hdds.utils.db.managed.ManagedSstFileWriter} is not used here because + * it stores sequence number 0 on every key and rejects duplicate user keys per file. Flushing a + * real RocksDB after each logical source batch yields SST files with global sequence numbers. + * Memtable flushes retain only the latest value per user key within each SST; competing versions + * appear across separate flushed files, matching production snapshot-diff inputs. + */ +@EnabledIfSystemProperty(named = ROCKS_TOOLS_NATIVE_PROPERTY, matches = "true") +class TestLatestVersionedKWayMergeIteratorOverSst { + + @TempDir + private Path tempDir; + + @BeforeAll + static void loadNativeLibrary() throws NativeLibraryNotLoadedException { + ManagedRawSSTFileReader.loadLibrary(); + } + + @Test + void testWorkedExampleOverRealSstFiles() throws Exception { + // Mirrors the unit-test "three-file worked example": cross-file k-way merge with competing + // versions, k1 latest-value-only, k2 delete-then-recreate across files. + Path dbDir = tempDir.resolve("worked-example-db"); + Files.createDirectories(dbDir); + Set<String> knownSstFiles = new HashSet<>(); + List<Path> sstFiles = new ArrayList<>(3); + + try (ManagedDBOptions dbOptions = new ManagedDBOptions(); + ManagedColumnFamilyOptions cfOptions = new ManagedColumnFamilyOptions(); + FlushOptions flushOptions = new FlushOptions()) { + dbOptions.setCreateIfMissing(true); + List<ColumnFamilyDescriptor> columnFamilyDescriptors = Collections.singletonList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOptions)); + List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(); + try (ManagedRocksDB db = ManagedRocksDB.open( + dbOptions, dbDir.toString(), columnFamilyDescriptors, columnFamilyHandles); + ColumnFamilyHandle cf = columnFamilyHandles.get(0)) { + + // Source A: latest k1 wins within the memtable before flush, plus k2. + rocksPut(db, cf, "k1", "v5"); + rocksPut(db, cf, "k1", "v10"); + rocksPut(db, cf, "k2", "v20"); + sstFiles.add(flushAndCopySst(db, dbDir, cf, flushOptions, knownSstFiles, "a")); + + // Source B: competing k1 version and a k2 tombstone. + rocksPut(db, cf, "k1", "v3"); + rocksPut(db, cf, "k1", "v15"); + rocksDelete(db, cf, "k2"); + sstFiles.add(flushAndCopySst(db, dbDir, cf, flushOptions, knownSstFiles, "b")); + + // Source C: winning k1/k2 values. + rocksPut(db, cf, "k1", "v1"); + rocksPut(db, cf, "k1", "v30"); + rocksPut(db, cf, "k2", "v25"); + sstFiles.add(flushAndCopySst(db, dbDir, cf, flushOptions, knownSstFiles, "c")); + } + } + + List<List<SourceRecord>> perSource = TestRawSstFileRecords.readFiles(sstFiles); + long k1VersionsAcrossFiles = perSource.stream() + .flatMap(List::stream) + .filter(record -> Arrays.equals(record.getUserKey(), keyBytes("k1"))) + .count(); + assertEquals(3, k1VersionsAcrossFiles, + "each flushed SST should contribute one surviving k1 version"); + long distinctK1Sequences = perSource.stream() + .flatMap(List::stream) + .filter(record -> Arrays.equals(record.getUserKey(), keyBytes("k1"))) + .mapToLong(SourceRecord::getSequence) + .distinct() + .count(); + assertEquals(3, distinctK1Sequences, + "k1 versions across SST files should carry distinct RocksDB sequence numbers"); + + List<MergedKeyValue> actual = mergeSstFiles(sstFiles.toArray(new Path[0])); + assertEquals(3, actual.size(), "expected k1 winner plus k2 tombstone and recreate value"); + + MergedKeyValue k1Winner = actual.get(0); + assertArrayEquals(keyBytes("k1"), k1Winner.getUserKey()); + assertEquals(LatestVersionedKWayMergeIterator.ROCKS_TYPE_VALUE, k1Winner.getValueType()); + assertArrayEquals(valueBytes("v30"), k1Winner.getValue()); + + MergedKeyValue k2Tombstone = actual.get(1); + assertArrayEquals(keyBytes("k2"), k2Tombstone.getUserKey()); + assertNotEquals(LatestVersionedKWayMergeIterator.ROCKS_TYPE_VALUE, k2Tombstone.getValueType()); + + MergedKeyValue k2Value = actual.get(2); + assertArrayEquals(keyBytes("k2"), k2Value.getUserKey()); + assertEquals(LatestVersionedKWayMergeIterator.ROCKS_TYPE_VALUE, k2Value.getValueType()); + assertArrayEquals(valueBytes("v25"), k2Value.getValue()); + assertTrue(k2Value.getSequence() > k2Tombstone.getSequence(), + "recreate value must be newer than the tombstone"); + } + + private Path flushAndCopySst(ManagedRocksDB db, Path dbDir, ColumnFamilyHandle cf, + FlushOptions flushOptions, Set<String> knownSstFiles, String label) + throws RocksDBException, IOException { + db.get().flush(flushOptions, cf); + Path newSst = findNewSstFile(dbDir, knownSstFiles); + Path dest = tempDir.resolve(label + "-" + newSst.getFileName()); + Files.copy(newSst, dest); + return dest; + } + + private static Path findNewSstFile(Path dbDir, Set<String> knownSstFiles) throws IOException { + try (Stream<Path> sstPaths = Files.list(dbDir)) { + List<Path> newFiles = sstPaths + .filter(path -> path.getFileName().toString().endsWith(".sst")) + .filter(path -> knownSstFiles.add(path.getFileName().toString())) + .sorted() + .collect(Collectors.toList()); + if (newFiles.size() != 1) { + throw new IllegalStateException( + "Expected exactly one new SST file under " + dbDir + ", found " + newFiles); + } + return newFiles.get(0); + } + } + + private List<MergedKeyValue> mergeSstFiles(Path... sstFiles) throws Exception { + List<MergedKeyValue> results = new ArrayList<>(); + try (LatestVersionedKWayMergeIterator iterator = + LatestVersionedKWayMergeIterator.overRawSstFiles(Arrays.asList(sstFiles))) { + while (iterator.hasNext()) { + results.add(iterator.next()); + } + } + return results; + } + + private static void rocksPut(ManagedRocksDB db, ColumnFamilyHandle cf, String key, String value) + throws RocksDBException { + db.get().put(cf, keyBytes(key), valueBytes(value)); + } + + private static void rocksDelete(ManagedRocksDB db, ColumnFamilyHandle cf, String key) + throws RocksDBException { + db.get().delete(cf, keyBytes(key)); + } + + private static byte[] keyBytes(String key) { + return key.getBytes(StandardCharsets.UTF_8); + } + + private static byte[] valueBytes(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } + +}
diff --git a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/TestRawSstFileRecords.java b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/TestRawSstFileRecords.java new file mode 100644 index 0000000..cd2cdc3 --- /dev/null +++ b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/TestRawSstFileRecords.java
@@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.utils.db; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.apache.hadoop.hdds.utils.db.managed.ManagedOptions; + +/** + * Test helper that reads all key versions from raw SST files. + */ +final class TestRawSstFileRecords { + + private static final int DEFAULT_READ_AHEAD_SIZE = 2 * 1024 * 1024; + + private TestRawSstFileRecords() { + } + + static List<SourceRecord> readFile(Path sstFile) throws IOException { + return readFile(sstFile, DEFAULT_READ_AHEAD_SIZE); + } + + static List<SourceRecord> readFile(Path sstFile, int readAheadSize) throws IOException { + List<SourceRecord> records = new ArrayList<>(); + try (ManagedOptions options = new ManagedOptions(); + ManagedRawSSTFileReader reader = new ManagedRawSSTFileReader( + options, sstFile.toAbsolutePath().toString(), readAheadSize); + ManagedRawSSTFileIterator<ManagedRawSSTFileIterator.KeyValue> iterator = + reader.newIterator(kv -> kv, null, null, IteratorType.KEY_AND_VALUE)) { + while (iterator.hasNext()) { + ManagedRawSSTFileIterator.KeyValue kv = iterator.next(); + byte[] key = copyBuffer(kv.getKey()); + byte[] value = kv.getType() == LatestVersionedKWayMergeIterator.ROCKS_TYPE_VALUE + ? copyBuffer(kv.getValue()) : null; + records.add(new SourceRecord(key, kv.getSequence().longValue(), kv.getType(), value)); + } + } + return records; + } + + static List<List<SourceRecord>> readFiles(List<Path> sstFiles) throws IOException { + return readFiles(sstFiles, DEFAULT_READ_AHEAD_SIZE); + } + + static List<List<SourceRecord>> readFiles(List<Path> sstFiles, int readAheadSize) + throws IOException { + List<List<SourceRecord>> perSource = new ArrayList<>(sstFiles.size()); + for (Path sstFile : sstFiles) { + perSource.add(readFile(sstFile, readAheadSize)); + } + return perSource; + } + + private static byte[] copyBuffer(CodecBuffer buffer) { + if (buffer == null) { + return null; + } + ByteBuffer byteBuffer = buffer.asReadOnlyByteBuffer(); + byte[] bytes = new byte[byteBuffer.remaining()]; + byteBuffer.get(bytes); + return bytes; + } + + static final class SourceRecord { + private final byte[] userKey; + private final long sequence; + private final int type; + private final byte[] value; + + SourceRecord(byte[] userKey, long sequence, int type, byte[] value) { + this.userKey = userKey; + this.sequence = sequence; + this.type = type; + this.value = value; + } + + byte[] getUserKey() { + return userKey; + } + + long getSequence() { + return sequence; + } + + int getType() { + return type; + } + + byte[] getValue() { + return value; + } + + @Override + public String toString() { + return "SourceRecord{key=" + Arrays.toString(userKey) + + ", seq=" + sequence + + ", type=" + type + + ", value=" + (value == null ? null : Arrays.toString(value)) + + '}'; + } + } +}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java index 3631df4..159551f 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java
@@ -17,10 +17,15 @@ package org.apache.hadoop.hdds.scm.container.balancer; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY; + import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.time.Duration; import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -28,11 +33,16 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.fs.DUFactory; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ContainerBalancerConfigurationProto; import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.container.ContainerManager; +import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; import org.apache.hadoop.hdds.scm.ha.SCMContext; import org.apache.hadoop.hdds.scm.ha.StatefulService; import org.apache.hadoop.hdds.scm.ha.StatefulServiceDefinition; +import org.apache.hadoop.hdds.scm.node.DatanodeInfo; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -470,6 +480,27 @@ private void validateConfiguration(ContainerBalancerConfiguration conf) " than ozone.scm.container.size"); } + if (conf.getMaxSizeEnteringTarget() > conf.getMaxSizeToMovePerIteration()) { + LOG.warn("hdds.container.balancer.size.entering.target.max {} should be " + + "less than or equal to hdds.container.balancer.size.moved.max" + + ".per.iteration {}", + conf.getMaxSizeEnteringTarget(), conf.getMaxSizeToMovePerIteration()); + throw new InvalidContainerBalancerConfigurationException( + "hdds.container.balancer.size.entering.target.max should be less " + + "than or equal to hdds.container.balancer.size.moved.max.per" + + ".iteration"); + } + if (conf.getMaxSizeLeavingSource() > conf.getMaxSizeToMovePerIteration()) { + LOG.warn("hdds.container.balancer.size.leaving.source.max {} should be " + + "less than or equal to hdds.container.balancer.size.moved.max" + + ".per.iteration {}", + conf.getMaxSizeLeavingSource(), conf.getMaxSizeToMovePerIteration()); + throw new InvalidContainerBalancerConfigurationException( + "hdds.container.balancer.size.leaving.source.max should be less " + + "than or equal to hdds.container.balancer.size.moved.max.per" + + ".iteration"); + } + // balancing interval should be greater than DUFactory refresh period DUFactory.Conf duConf = ozoneConfiguration.getObject(DUFactory.Conf.class); long refreshPeriod = duConf.getRefreshPeriod().toMillis(); @@ -510,6 +541,133 @@ private void validateConfiguration(ContainerBalancerConfiguration conf) validateNodeList(conf.getIncludeNodes(), "included"); validateNodeList(conf.getExcludeNodes(), "excluded"); + validateIncludeExcludeLists(conf); + validateIncludeContainersExist(conf); + validateEligibleDatanodePool(conf); + } + + /** + * Rejects include lists that are fully covered by the corresponding exclude + * lists, which would leave no datanodes or containers to balance. + */ + private void validateIncludeExcludeLists(ContainerBalancerConfiguration conf) + throws InvalidContainerBalancerConfigurationException { + Set<String> includeNodes = conf.getIncludeNodes(); + Set<String> excludeNodes = conf.getExcludeNodes(); + if (!includeNodes.isEmpty() && !excludeNodes.isEmpty()) { + boolean allIncludedNodesExcluded = true; + for (String includedNode : includeNodes) { + if (!isIncludedNodeExcluded(includedNode, excludeNodes)) { + allIncludedNodesExcluded = false; + break; + } + } + if (allIncludedNodesExcluded) { + throw new InvalidContainerBalancerConfigurationException( + "include-datanodes is a subset of exclude-datanodes, no datanode can participate in balancing."); + } + } + + Set<ContainerID> includeContainers = conf.getIncludeContainers(); + Set<ContainerID> excludeContainers = conf.getExcludeContainers(); + if (!includeContainers.isEmpty() && excludeContainers.containsAll(includeContainers)) { + throw new InvalidContainerBalancerConfigurationException( + "include-containers is a subset of exclude-containers, no container can be selected for balancing."); + } + } + + private boolean isIncludedNodeExcluded(String includedNode, Set<String> excludeNodes) { + if (excludeNodes.contains(includedNode)) { + return true; + } + for (DatanodeDetails dn : scm.getScmNodeManager().getNodesByAddress(includedNode)) { + if (excludeNodes.contains(dn.getHostName()) || excludeNodes.contains(dn.getIpAddress())) { + return true; + } + } + return false; + } + + /** + * Rejects non-empty include-containers lists when any listed container ID + * does not exist in SCM. + */ + private void validateIncludeContainersExist(ContainerBalancerConfiguration conf) + throws InvalidContainerBalancerConfigurationException { + Set<ContainerID> includeContainers = conf.getIncludeContainers(); + if (includeContainers.isEmpty()) { + return; + } + + ContainerManager containerManager = scm.getContainerManager(); + List<ContainerID> missingContainers = new ArrayList<>(); + for (ContainerID containerID : includeContainers) { + try { + containerManager.getContainer(containerID); + } catch (ContainerNotFoundException e) { + missingContainers.add(containerID); + } + } + + if (!missingContainers.isEmpty()) { + throw new InvalidContainerBalancerConfigurationException( + "Container Balancer cannot start: included container ID(s) " + missingContainers + + " do not exist in SCM."); + } + } + + /** + * Validates that enough healthy, in-service datanodes are eligible and that + * {@link ContainerBalancerConfiguration#getMaxDatanodesRatioToInvolvePerIteration()} + * allows at least one source and one target datanode per iteration. + */ + private void validateEligibleDatanodePool(ContainerBalancerConfiguration conf) + throws InvalidContainerBalancerConfigurationException { + int eligibleCount = countEligibleDatanodes(conf); + if (eligibleCount < 2) { + throw new InvalidContainerBalancerConfigurationException(String.format( + "Container Balancer found %d eligible datanode(s) but requires at least 2.", + eligibleCount)); + } + int maxDatanodesToInvolve = conf.computeMaxDatanodesToInvolvePerIteration(eligibleCount); + if (maxDatanodesToInvolve < 2) { + throw new InvalidContainerBalancerConfigurationException(String.format( + "max-datanodes-percentage-to-involve-per-iteration=%d allows at most " + + "%d datanode(s) per iteration with %d eligible datanode(s), " + + "but at least 2 are required for a source and target datanode " + + "pair.", + conf.getMaxDatanodesPercentageToInvolvePerIteration(), + maxDatanodesToInvolve, eligibleCount)); + } + } + + /** + * Counts healthy, in-service datanodes that can participate in balancing after + * applying include/exclude datanode configuration. + */ + private int countEligibleDatanodes(ContainerBalancerConfiguration conf) { + Set<String> excludeNodes = conf.getExcludeNodes(); + Set<String> includeNodes = conf.getIncludeNodes(); + List<DatanodeInfo> healthyNodes = scm.getScmNodeManager().getNodes(IN_SERVICE, HEALTHY); + int eligibleCount = 0; + for (DatanodeDetails datanode : healthyNodes) { + if (!shouldExcludeDatanode(datanode, excludeNodes, includeNodes)) { + eligibleCount++; + } + } + return eligibleCount; + } + + static boolean shouldExcludeDatanode(DatanodeDetails datanode, + Set<String> excludeNodes, Set<String> includeNodes) { + if (excludeNodes.contains(datanode.getHostName()) || + excludeNodes.contains(datanode.getIpAddress())) { + return true; + } else if (!includeNodes.isEmpty()) { + return !includeNodes.contains(datanode.getHostName()) && + !includeNodes.contains(datanode.getIpAddress()); + } + return false; } public ContainerBalancerMetrics getMetrics() {
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java index 3855425..bb2b4c8 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java
@@ -85,7 +85,6 @@ public class ContainerBalancerTask implements Runnable { private ContainerBalancer containerBalancer; private final SCMContext scmContext; private int totalNodesInCluster; - private double maxDatanodesRatioToInvolvePerIteration; private long maxSizeToMovePerIteration; private int countDatanodesInvolvedPerIteration; private long sizeScheduledForMoveInLatestIteration; @@ -537,8 +536,6 @@ private boolean initializeIteration() { return false; } - this.maxDatanodesRatioToInvolvePerIteration = - config.getMaxDatanodesRatioToInvolvePerIteration(); this.maxSizeToMovePerIteration = config.getMaxSizeToMovePerIteration(); this.excludeNodes = config.getExcludeNodes(); @@ -939,7 +936,7 @@ private boolean reachedMaxSizeToMovePerIteration() { private boolean adaptWhenNearingIterationLimits() { // check if we're nearing max datanodes to involve int maxDatanodesToInvolve = - (int) (maxDatanodesRatioToInvolvePerIteration * totalNodesInCluster); + config.computeMaxDatanodesToInvolvePerIteration(totalNodesInCluster); if (countDatanodesInvolvedPerIteration + 1 == maxDatanodesToInvolve) { /* We're one datanode away from reaching the limit. Restrict potential targets to targets that have already been selected. @@ -966,7 +963,7 @@ private boolean adaptWhenNearingIterationLimits() { private boolean adaptOnReachingIterationLimits() { // check if we've reached max datanodes to involve limit int maxDatanodesToInvolve = - (int) (maxDatanodesRatioToInvolvePerIteration * totalNodesInCluster); + config.computeMaxDatanodesToInvolvePerIteration(totalNodesInCluster); if (countDatanodesInvolvedPerIteration == maxDatanodesToInvolve) { // restrict both to already selected sources and targets findTargetStrategy.resetPotentialTargets(selectedTargets); @@ -1172,14 +1169,7 @@ private List<DatanodeUsageInfo> getPotentialSources() { * @return true if Datanode should be excluded, else false */ private boolean shouldExcludeDatanode(DatanodeDetails datanode) { - if (excludeNodes.contains(datanode.getHostName()) || - excludeNodes.contains(datanode.getIpAddress())) { - return true; - } else if (!includeNodes.isEmpty()) { - return !includeNodes.contains(datanode.getHostName()) && - !includeNodes.contains(datanode.getIpAddress()); - } - return false; + return ContainerBalancer.shouldExcludeDatanode(datanode, excludeNodes, includeNodes); } /**
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java index 30eb83a..35512b2 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
@@ -17,6 +17,8 @@ package org.apache.hadoop.hdds.scm.pipeline; +import static org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name.RATIS_DATASTREAM; + import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.util.ArrayList; @@ -40,6 +42,7 @@ import org.apache.hadoop.hdds.scm.pipeline.leader.choose.algorithms.LeaderChoosePolicy; import org.apache.hadoop.hdds.scm.pipeline.leader.choose.algorithms.LeaderChoosePolicyFactory; import org.apache.hadoop.hdds.server.events.EventPublisher; +import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.protocol.commands.ClosePipelineCommand; import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode; import org.apache.hadoop.ozone.protocol.commands.CreatePipelineCommand; @@ -64,6 +67,7 @@ public class RatisPipelineProvider private final SCMContext scmContext; private final long containerSizeBytes; private final long minRatisVolumeSizeBytes; + private final boolean isRatisStreamingEnabled; @VisibleForTesting public RatisPipelineProvider(NodeManager nodeManager, @@ -90,6 +94,9 @@ public RatisPipelineProvider(NodeManager nodeManager, ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN, ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN_DEFAULT, StorageUnit.BYTES); + this.isRatisStreamingEnabled = conf.getBoolean( + OzoneConfigKeys.HDDS_CONTAINER_RATIS_DATASTREAM_ENABLED, + OzoneConfigKeys.HDDS_CONTAINER_RATIS_DATASTREAM_ENABLED_DEFAULT); try { leaderChoosePolicy = LeaderChoosePolicyFactory .getPolicy(conf, nodeManager, stateManager); @@ -163,17 +170,8 @@ public synchronized Pipeline create(RatisReplicationConfig replicationConfig, dns = pickNodesNotUsed(replicationConfig, minRatisVolumeSizeBytes, containerSizeBytes); break; case THREE: - List<DatanodeDetails> excludeDueToEngagement = filterPipelineEngagement(); - if (!excludeDueToEngagement.isEmpty()) { - if (excludedNodes.isEmpty()) { - excludedNodes = excludeDueToEngagement; - } else { - excludedNodes.addAll(excludeDueToEngagement); - } - } - dns = placementPolicy.chooseDatanodes(excludedNodes, - favoredNodes, factor.getNumber(), minRatisVolumeSizeBytes, - containerSizeBytes); + dns = chooseThreeFactorDatanodes(excludedNodes, favoredNodes, factor.getNumber()); + break; default: throw new IllegalStateException("Unknown factor: " + factor.name()); @@ -224,18 +222,45 @@ public Pipeline createForRead( .build(); } - private List<DatanodeDetails> filterPipelineEngagement() { + private List<DatanodeDetails> chooseThreeFactorDatanodes( + List<DatanodeDetails> excludedNodes, List<DatanodeDetails> favoredNodes, int requiredNode) + throws IOException { final NodeManager nodeManager = getNodeManager(); final PipelineStateManager stateManager = getPipelineStateManager(); final List<DatanodeDetails> healthyNodes = nodeManager.getNodes(NodeStatus.inServiceHealthy()); - final List<DatanodeDetails> excluded = new ArrayList<>(); + excludedNodes = excludedNodes.isEmpty() ? null : excludedNodes; + List<DatanodeDetails> additionalExcludedNodes = null; for (DatanodeDetails d : healthyNodes) { final int count = PipelinePlacementPolicy.currentRatisThreePipelineCount(nodeManager, stateManager, d); if (count >= nodeManager.pipelineLimit(d)) { - excluded.add(d); + if (excludedNodes == null) { + excludedNodes = new ArrayList<>(); + } + excludedNodes.add(d); + } else if (isRatisStreamingEnabled && !d.hasPort(RATIS_DATASTREAM)) { + if (additionalExcludedNodes == null) { + additionalExcludedNodes = new ArrayList<>(); + } + additionalExcludedNodes.add(d); } } - return excluded; + // If the cluster does not support Ratis streaming, or if all nodes support it, no fallback will occur. + if (additionalExcludedNodes != null) { + if (excludedNodes != null) { + additionalExcludedNodes.addAll(excludedNodes); + } + try { + return placementPolicy.chooseDatanodes(additionalExcludedNodes, + favoredNodes, requiredNode, minRatisVolumeSizeBytes, + containerSizeBytes); + } catch (SCMException scmException) { + LOG.debug("Failed to allocate datanodes with strict exclusion (non-streaming nodes excluded)." + + " Falling back.", scmException); + } + } + return placementPolicy.chooseDatanodes(excludedNodes, + favoredNodes, requiredNode, minRatisVolumeSizeBytes, + containerSizeBytes); } /**
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java index 7324ae3..acb3e64 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -1235,9 +1235,9 @@ public StartContainerBalancerResponseProto startContainerBalancer( int mdti = maxDatanodesPercentageToInvolvePerIteration.get(); auditMap.put("maxDatanodesPercentageToInvolvePerIteration", String.valueOf(mdti)); - if (mdti < 0 || mdti > 100) { + if (mdti <= 0 || mdti > 100) { throw new IOException("Max Datanodes Percentage To Involve Per Iteration" + - "should be specified in the range [0, 100]"); + "should be specified in the range (0, 100]"); } cbc.setMaxDatanodesPercentageToInvolvePerIteration(mdti); }
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java index 4864f23..8931c9d 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java
@@ -19,6 +19,8 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_NODE_REPORT_INTERVAL; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -36,20 +38,27 @@ import com.google.protobuf.ByteString; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.container.ContainerManager; +import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; import org.apache.hadoop.hdds.scm.ha.SCMContext; import org.apache.hadoop.hdds.scm.ha.SCMServiceManager; import org.apache.hadoop.hdds.scm.ha.StatefulServiceStateManager; import org.apache.hadoop.hdds.scm.ha.StatefulServiceStateManagerImpl; +import org.apache.hadoop.hdds.scm.node.DatanodeInfo; import org.apache.hadoop.hdds.scm.node.NodeManager; import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.GenericTestUtils.LogCapturer; import org.junit.jupiter.api.BeforeEach; @@ -67,6 +76,7 @@ public class TestContainerBalancer { private ContainerBalancer containerBalancer; private StorageContainerManager scm; + private NodeManager nodeManager; private ContainerBalancerConfiguration balancerConfiguration; private Map<String, ByteString> serviceToConfigMap = new HashMap<>(); private OzoneConfiguration conf; @@ -95,6 +105,9 @@ public void setup() throws IOException, NodeNotFoundException, GenericTestUtils.setLogLevel(ContainerBalancer.class, Level.DEBUG); when(scm.getScmNodeManager()).thenReturn(mock(NodeManager.class)); + nodeManager = scm.getScmNodeManager(); + List<DatanodeInfo> eligibleDatanodes = createEligibleDatanodes(10); + when(nodeManager.getNodes(IN_SERVICE, HEALTHY)).thenReturn(eligibleDatanodes); when(scm.getScmContext()).thenReturn(SCMContext.emptyContext()); when(scm.getConfiguration()).thenReturn(conf); when(scm.getStatefulServiceStateManager()).thenReturn(serviceStateManager); @@ -296,12 +309,11 @@ public void testGetBalancerStatusInfo() throws Exception { @Test public void testStartBalancerWithInvalidNodes() throws Exception { - NodeManager nm = scm.getScmNodeManager(); String validHost = "1.2.3.4"; String invalidHost = "invalid-host-name"; - when(nm.getNodesByAddress(invalidHost)).thenReturn(Collections.emptyList()); - when(nm.getNodesByAddress(validHost)).thenReturn(Collections.singletonList(mock(DatanodeDetails.class))); + when(nodeManager.getNodesByAddress(invalidHost)).thenReturn(Collections.emptyList()); + when(nodeManager.getNodesByAddress(validHost)).thenReturn(Collections.singletonList(mock(DatanodeDetails.class))); // Test invalid includeNodes balancerConfiguration.setIncludeNodes(invalidHost); @@ -321,7 +333,7 @@ public void testStartBalancerWithInvalidNodes() throws Exception { // Test a valid case balancerConfiguration.setExcludeNodes(""); - balancerConfiguration.setIncludeNodes(validHost); + balancerConfiguration.setIncludeNodes(""); assertDoesNotThrow(() -> startBalancer(balancerConfiguration)); assertSame(ContainerBalancerTask.Status.RUNNING, containerBalancer.getBalancerStatus()); @@ -371,6 +383,93 @@ public void testGetBalancerStatusInfoAfterScmStop() throws Exception { assertNotNull(statusInfo.getStoppedAt()); } + /** + * Tests new startup validation for conflicting include/exclude lists. + */ + @Test + public void testRejectConflictingIncludeExcludeLists() throws Exception { + when(nodeManager.getNodesByAddress("dn0")) + .thenReturn(Collections.singletonList(mock(DatanodeDetails.class))); + when(nodeManager.getNodesByAddress("dn1")) + .thenReturn(Collections.singletonList(mock(DatanodeDetails.class))); + balancerConfiguration.setIncludeNodes("dn0,dn1"); + balancerConfiguration.setExcludeNodes("dn0,dn1"); + + InvalidContainerBalancerConfigurationException ex = + assertThrows(InvalidContainerBalancerConfigurationException.class, + () -> containerBalancer.startBalancer(balancerConfiguration)); + assertThat(ex.getMessage()).contains( + "include-datanodes is a subset of exclude-datanodes"); + assertSame(ContainerBalancerTask.Status.STOPPED, containerBalancer.getBalancerStatus()); + + balancerConfiguration.setIncludeNodes(""); + balancerConfiguration.setExcludeNodes(""); + balancerConfiguration.setIncludeContainers("1, 2"); + balancerConfiguration.setExcludeContainers("1,2,3"); + ex = assertThrows(InvalidContainerBalancerConfigurationException.class, + () -> containerBalancer.startBalancer(balancerConfiguration)); + assertThat(ex.getMessage()).contains( + "include-containers is a subset of exclude-containers"); + assertSame(ContainerBalancerTask.Status.STOPPED, containerBalancer.getBalancerStatus()); + } + + /** + * Tests new startup validation for include-containers, datanode pool, and + * size limits. + */ + @Test + public void testRejectInvalidStartupConfiguration() throws Exception { + ContainerManager containerManager = mock(ContainerManager.class); + when(scm.getContainerManager()).thenReturn(containerManager); + when(containerManager.getContainer(any(ContainerID.class))) + .thenThrow(new ContainerNotFoundException(ContainerID.valueOf(1))); + + balancerConfiguration.setIncludeContainers("1"); + InvalidContainerBalancerConfigurationException ex = + assertThrows(InvalidContainerBalancerConfigurationException.class, + () -> containerBalancer.startBalancer(balancerConfiguration)); + assertThat(ex.getMessage()).contains("do not exist in SCM"); + assertSame(ContainerBalancerTask.Status.STOPPED, containerBalancer.getBalancerStatus()); + + balancerConfiguration.setIncludeContainers(""); + List<DatanodeInfo> fiveDatanodes = createEligibleDatanodes(5); + when(nodeManager.getNodes(IN_SERVICE, HEALTHY)).thenReturn(fiveDatanodes); + balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(20); + ex = assertThrows(InvalidContainerBalancerConfigurationException.class, + () -> containerBalancer.startBalancer(balancerConfiguration)); + assertThat(ex.getMessage()).contains("at least 2 are required for a source and target datanode pair."); + assertSame(ContainerBalancerTask.Status.STOPPED, containerBalancer.getBalancerStatus()); + + balancerConfiguration.setMaxSizeToMovePerIteration(100 * OzoneConsts.GB); + balancerConfiguration.setMaxSizeEnteringTarget(200 * OzoneConsts.GB); + ex = assertThrows(InvalidContainerBalancerConfigurationException.class, + () -> containerBalancer.startBalancer(balancerConfiguration)); + assertThat(ex.getMessage()).contains( + "hdds.container.balancer.size.entering.target.max should be less than or " + + "equal to hdds.container.balancer.size.moved.max.per.iteration"); + assertSame(ContainerBalancerTask.Status.STOPPED, containerBalancer.getBalancerStatus()); + + balancerConfiguration.setMaxSizeEnteringTarget(100 * OzoneConsts.GB); + balancerConfiguration.setMaxSizeLeavingSource(200 * OzoneConsts.GB); + ex = assertThrows(InvalidContainerBalancerConfigurationException.class, + () -> containerBalancer.startBalancer(balancerConfiguration)); + assertThat(ex.getMessage()).contains( + "hdds.container.balancer.size.leaving.source.max should be less than or " + + "equal to hdds.container.balancer.size.moved.max.per.iteration"); + assertSame(ContainerBalancerTask.Status.STOPPED, containerBalancer.getBalancerStatus()); + } + + private static List<DatanodeInfo> createEligibleDatanodes(int count) { + List<DatanodeInfo> datanodes = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + DatanodeInfo datanode = mock(DatanodeInfo.class); + when(datanode.getHostName()).thenReturn("dn" + i); + when(datanode.getIpAddress()).thenReturn("10.0.0." + i); + datanodes.add(datanode); + } + return datanodes; + } + private void startBalancer(ContainerBalancerConfiguration config) throws IllegalContainerBalancerStateException, IOException, InvalidContainerBalancerConfigurationException, TimeoutException {
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java index bf352f2..5017a65 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java
@@ -34,7 +34,9 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Random; import java.util.Set; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import org.apache.hadoop.hdds.HddsConfigKeys; @@ -57,10 +59,12 @@ import org.apache.hadoop.hdds.scm.ha.SCMHAManager; import org.apache.hadoop.hdds.scm.ha.SCMHAManagerStub; import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; +import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl; import org.apache.hadoop.hdds.scm.node.NodeStatus; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.OzoneConfigKeys; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.Test; @@ -93,10 +97,21 @@ public void init(int maxPipelinePerNode, OzoneConfiguration conf) init(maxPipelinePerNode, conf, testDir); } + public void initWithNodes(int maxPipelinePerNode, OzoneConfiguration conf, List<DatanodeDetails> nodes, int count) + throws Exception { + conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath()); + nodeManager = new MockNodeManager(new NetworkTopologyImpl(new OzoneConfiguration()), nodes, false, count); + initializeCommonState(maxPipelinePerNode, conf); + } + public void init(int maxPipelinePerNode, OzoneConfiguration conf, File dir) throws Exception { conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, dir.getAbsolutePath()); - dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get()); nodeManager = new MockNodeManager(true, nodeCount); + initializeCommonState(maxPipelinePerNode, conf); + } + + private void initializeCommonState(int maxPipelinePerNode, OzoneConfiguration conf) throws Exception { + dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get()); nodeManager.setNumPipelinePerDatanode(maxPipelinePerNode); long containerSize = (long) conf.getStorageSize( ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, @@ -246,6 +261,79 @@ public void testCreateFactorTHREEPipelineWithSameDatanodes() assertEquals(pipeline2.getNodeSet(), pipeline3.getNodeSet()); } + private DatanodeDetails createDatanodeDetails(boolean supportRatisStreaming) { + Random random = ThreadLocalRandom.current(); + String ipAddress = random.nextInt(256) + + "." + random.nextInt(256) + + "." + random.nextInt(256) + + "." + random.nextInt(256); + + DatanodeDetails.Builder dn = DatanodeDetails.newBuilder() + .setID(DatanodeID.randomID()) + .setHostName("localhost" + "-" + ipAddress) + .setIpAddress(ipAddress) + .setNetworkLocation(null) + .setPersistedOpState(HddsProtos.NodeOperationalState.IN_SERVICE) + .setPersistedOpStateExpiry(0); + + for (DatanodeDetails.Port.Name name : DatanodeDetails.Port.Name.values()) { + if (!supportRatisStreaming && name == DatanodeDetails.Port.Name.RATIS_DATASTREAM) { + continue; + } + dn.addPort(DatanodeDetails.newPort(name, 0)); + } + return dn.build(); + } + + @Test + public void testCreatePipelinePrioritizesRatisStreamingNodes() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.setBoolean( + OzoneConfigKeys.HDDS_CONTAINER_RATIS_DATASTREAM_ENABLED, true); + List<DatanodeDetails> nodes = new ArrayList<>(); + // Add 3 nodes WITH RATIS_DATASTREAM + for (int i = 0; i < 3; i++) { + nodes.add(createDatanodeDetails(true)); + } + // Add 3 nodes WITHOUT RATIS_DATASTREAM + for (int i = 0; i < 3; i++) { + nodes.add(createDatanodeDetails(false)); + } + + initWithNodes(1, conf, nodes, 3); + Pipeline pipeline = provider.create(RatisReplicationConfig.getInstance(ReplicationFactor.THREE)); + assertEquals(3, pipeline.getNodes().size()); + for (DatanodeDetails dn : pipeline.getNodes()) { + assertTrue(dn.hasPort(DatanodeDetails.Port.Name.RATIS_DATASTREAM), + "Pipeline should only contain datanodes with RATIS_DATASTREAM when available"); + } + } + + @Test + public void testCreatePipelineFallsBackWhenNotEnoughRatisStreamingNodes() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.setBoolean( + OzoneConfigKeys.HDDS_CONTAINER_RATIS_DATASTREAM_ENABLED, true); + List<DatanodeDetails> nodes = new ArrayList<>(); + // Add 2 nodes WITH RATIS_DATASTREAM + for (int i = 0; i < 2; i++) { + nodes.add(createDatanodeDetails(true)); + } + // Add 1 node WITHOUT RATIS_DATASTREAM + nodes.add(createDatanodeDetails(false)); + + initWithNodes(1, conf, nodes, 3); + Pipeline pipeline = provider.create(RatisReplicationConfig.getInstance(ReplicationFactor.THREE)); + assertEquals(3, pipeline.getNodes().size()); + + long streamingNodeCount = pipeline.getNodes().stream() + .filter(dn -> dn.hasPort(DatanodeDetails.Port.Name.RATIS_DATASTREAM)) + .count(); + + assertEquals(2, streamingNodeCount, + "Pipeline should contain exactly 2 nodes with RATIS_DATASTREAM as fallback was required"); + } + @Test public void testCreatePipelinesDnExclude() throws Exception {
diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java index 2cab942..09de9e2 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java
@@ -143,10 +143,8 @@ public void execute(ScmClient scmClient) throws IOException { System.out.println("Container Balancer started successfully."); } else { String reason = ""; - System.err.println("Failed to start Container Balancer."); if (response.hasMessage()) { reason = response.getMessage(); - System.err.printf("Failure reason: %s%n", reason); } throw new IOException("Failed to start Container Balancer. " + reason); }
diff --git a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java index 613ca37..407785d 100644 --- a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java +++ b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java
@@ -596,8 +596,8 @@ public void testContainerBalancerStartSubcommandWhenBalancerIsRunning() .setStart(false) .setMessage("") .build()); - assertThrows(IOException.class, () -> startCmd.execute(scmClient)); - assertThat(err.get()).containsPattern(FAILED_TO_START); + IOException ex = assertThrows(IOException.class, () -> startCmd.execute(scmClient)); + assertThat(ex.getMessage()).containsPattern(FAILED_TO_START); } @Test
diff --git a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/om/ContainerToKeyMapping.java b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/om/ContainerToKeyMapping.java index 589bd98..14559da 100644 --- a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/om/ContainerToKeyMapping.java +++ b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/om/ContainerToKeyMapping.java
@@ -52,7 +52,11 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmMultipartPartInfo; +import org.apache.hadoop.ozone.om.helpers.OmMultipartPartKey; +import org.apache.hadoop.ozone.om.helpers.OmMultipartUpload; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo; import picocli.CommandLine; @@ -100,6 +104,7 @@ public class ContainerToKeyMapping extends AbstractSubcommand implements Callabl private Table<String, OmKeyInfo> openFileTable; private Table<String, OmKeyInfo> openKeyTable; private Table<String, OmMultipartKeyInfo> multipartInfoTable; + private Table<OmMultipartPartKey, OmMultipartPartInfo> multipartPartsTable; private DBStore dirTreeDbStore; private Table<Long, String> dirTreeTable; // Cache volume IDs to avoid repeated lookups @@ -138,6 +143,7 @@ public Void call() throws Exception { openFileTable = OMDBDefinition.OPEN_FILE_TABLE_DEF.getTable(omDbStore, CacheType.NO_CACHE); openKeyTable = OMDBDefinition.OPEN_KEY_TABLE_DEF.getTable(omDbStore, CacheType.NO_CACHE); multipartInfoTable = OMDBDefinition.MULTIPART_INFO_TABLE_DEF.getTable(omDbStore, CacheType.NO_CACHE); + multipartPartsTable = OMDBDefinition.MULTIPART_PARTS_TABLE_DEF.getTable(omDbStore, CacheType.NO_CACHE); retrieve(dbPath, writer, containerIDs); } catch (Exception e) { @@ -313,11 +319,18 @@ private void processMultipartUpload(Set<Long> containerIds, Map<Long, List<Strin String dbKey = entry.getKey(); OmMultipartKeyInfo mpuInfo = entry.getValue(); - // Collect all target containers that have parts of this MPU + // Collect all target containers that have parts of this MPU. Legacy + // (schemaVersion 0) MPUs embed their parts in the multipartInfoTable + // value, whereas split (schemaVersion 1) MPUs store each part as a + // separate row in the multipartPartsTable. Set<Long> matchedContainers = new HashSet<>(); - for (PartKeyInfo partKeyInfo : mpuInfo.getPartKeyInfoMap()) { - OmKeyInfo partKey = OmKeyInfo.getFromProtobuf(partKeyInfo.getPartKeyInfo()); - matchedContainers.addAll(getKeyContainers(partKey, containerIds)); + if (mpuInfo.getSchemaVersion() == OmMultipartKeyInfo.SPLIT_PARTS_TABLE_SCHEMA_VERSION) { + matchedContainers.addAll(getSplitPartContainers(dbKey, containerIds)); + } else { + for (PartKeyInfo partKeyInfo : mpuInfo.getPartKeyInfoMap()) { + OmKeyInfo partKey = OmKeyInfo.getFromProtobuf(partKeyInfo.getPartKeyInfo()); + matchedContainers.addAll(getKeyContainers(partKey, containerIds)); + } } if (!matchedContainers.isEmpty()) { @@ -332,8 +345,48 @@ private void processMultipartUpload(Set<Long> containerIds, Map<Long, List<Strin } private Set<Long> getKeyContainers(OmKeyInfo keyInfo, Set<Long> targetContainerIds) { + return getContainers(keyInfo.getKeyLocationVersions(), targetContainerIds); + } + + /** + * Scans the split multipartPartsTable for all parts belonging to the given + * multipart upload (its uploadId is the last path component of the + * multipartInfoTable db key) and returns the target containers referenced by + * those parts' block locations. + */ + private Set<Long> getSplitPartContainers(String multipartInfoDbKey, Set<Long> targetContainerIds) { + Set<Long> matchedContainers = new HashSet<>(); + String uploadId; + try { + uploadId = OmMultipartUpload.from(multipartInfoDbKey).getUploadId(); + } catch (IllegalArgumentException e) { + err().println("Invalid multipartInfoTable key " + multipartInfoDbKey + ", " + e); + return matchedContainers; + } + OmMultipartPartKey prefix = OmMultipartPartKey.prefix(uploadId); + try (TableIterator<OmMultipartPartKey, Table.KeyValue<OmMultipartPartKey, OmMultipartPartInfo>> + partIterator = multipartPartsTable.iterator(prefix)) { + while (partIterator.hasNext()) { + Table.KeyValue<OmMultipartPartKey, OmMultipartPartInfo> partEntry = partIterator.next(); + OmMultipartPartKey partKey = partEntry.getKey(); + // Prefix iteration can overshoot into the next upload's rows; stop then. + if (!uploadId.equals(partKey.getUploadId())) { + break; + } + if (partKey.hasPartNumber()) { + matchedContainers.addAll( + getContainers(partEntry.getValue().getKeyLocationInfos(), targetContainerIds)); + } + } + } catch (Exception e) { + err().println("Exception occurred reading multipartPartsTable for upload " + uploadId + ", " + e); + } + return matchedContainers; + } + + private Set<Long> getContainers(List<OmKeyLocationInfoGroup> locationVersions, Set<Long> targetContainerIds) { Set<Long> keyContainers = new HashSet<>(); - keyInfo.getKeyLocationVersions().forEach( + locationVersions.forEach( e -> e.getLocationList().forEach( blk -> { long cid = blk.getBlockID().getContainerID();
diff --git a/hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/om/TestContainerToKeyMapping.java b/hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/om/TestContainerToKeyMapping.java index 4cad62c..3a2e396 100644 --- a/hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/om/TestContainerToKeyMapping.java +++ b/hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/om/TestContainerToKeyMapping.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.debug.OzoneDebug; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; @@ -41,6 +42,8 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmMultipartPartInfo; +import org.apache.hadoop.ozone.om.helpers.OmMultipartPartKey; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo; @@ -76,6 +79,7 @@ public class TestContainerToKeyMapping { private static final long CONTAINER_ID_2 = 2L; private static final long CONTAINER_ID_3 = 3L; private static final long CONTAINER_ID_4 = 4L; + private static final long CONTAINER_ID_5 = 5L; private static final long UNREFERENCED_FILE_ID = 500L; private static final long MISSING_DIR_ID = 999L; // Non-existent parent private static final long OPEN_FILE_ID = 600L; @@ -190,6 +194,18 @@ public void testContainerToKeyMappingWithMPUOnlyFileNames() { } @Test + public void testContainerToKeyMappingWithSplitSchemaMPU() { + int exitCode = execute("--containers", String.valueOf(CONTAINER_ID_5), "--in-progress"); + assertEquals(0, exitCode); + + String output = outWriter.toString(); + + assertThat(output).contains("\"" + CONTAINER_ID_5 + "\""); + assertThat(output).contains("\"openKeys\""); + assertThat(output).contains("/vol1/obs-bucket/splitMpuKey/split-upload-id"); + } + + @Test public void testNonExistentContainer() { long nonExistentContainerId = 999L; @@ -292,6 +308,7 @@ private void createTestData() throws Exception { // Create MPU (multipart upload) for OBS bucket with parts in container 5 createMultipartUpload(); + createSplitSchemaMultipartUpload(); } /** @@ -340,6 +357,46 @@ private void createMultipartUpload() throws Exception { } /** + * Helper method to create a split-schema multipart upload with parts in + * multipartPartsTable. + */ + private void createSplitSchemaMultipartUpload() throws Exception { + String mpuKeyName = "splitMpuKey"; + String uploadId = "split-upload-id"; + + OmKeyInfo part1Info = new OmKeyInfo.Builder(createOBSKeyInfo( + mpuKeyName + "/" + uploadId + "/part-1", MPU_PART1_ID + 10, CONTAINER_ID_5)) + .addMetadata(OzoneConsts.ETAG, "etag-1") + .build(); + OmMultipartPartInfo partInfo1 = OmMultipartPartInfo.from( + mpuKeyName + "/" + uploadId + "/part-1", 1, part1Info); + + OmKeyInfo part2Info = new OmKeyInfo.Builder(createOBSKeyInfo( + mpuKeyName + "/" + uploadId + "/part-2", MPU_PART2_ID + 10, CONTAINER_ID_5)) + .addMetadata(OzoneConsts.ETAG, "etag-2") + .build(); + OmMultipartPartInfo partInfo2 = OmMultipartPartInfo.from( + mpuKeyName + "/" + uploadId + "/part-2", 2, part2Info); + + omMetadataManager.getMultipartPartsTable().put(OmMultipartPartKey.of(uploadId, 1), partInfo1); + omMetadataManager.getMultipartPartsTable().put(OmMultipartPartKey.of(uploadId, 2), partInfo2); + + OmMultipartKeyInfo mpuInfo = new OmMultipartKeyInfo.Builder() + .setUploadID(uploadId) + .setCreationTime(System.currentTimeMillis()) + .setReplicationConfig(StandaloneReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)) + .setSchemaVersion(OmMultipartKeyInfo.SPLIT_PARTS_TABLE_SCHEMA_VERSION) + .setObjectID(MPU_KEY_ID + 10) + .setParentID(0) + .setUpdateID(1) + .build(); + + String mpuKey = omMetadataManager.getMultipartKey( + VOLUME_NAME, OBS_BUCKET_NAME, mpuKeyName, uploadId); + omMetadataManager.getMultipartInfoTable().put(mpuKey, mpuInfo); + } + + /** * Helper method to create OmKeyInfo with a block in specified container (FSO). */ private OmKeyInfo createKeyInfo(String keyName, long objectId, long parentId, long containerId) { @@ -386,6 +443,8 @@ private OmKeyInfo createOBSKeyInfo(String keyName, long objectId, long container .setDataSize(1024) .setObjectID(objectId) .setUpdateID(1) + .setCreationTime(System.currentTimeMillis()) + .setModificationTime(System.currentTimeMillis()) .addOmKeyLocationInfoGroup(locationGroup) .build(); }
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OmConfig.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OmConfig.java index b8a60f9..c981afd 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OmConfig.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OmConfig.java
@@ -184,6 +184,18 @@ public class OmConfig extends ReconfigurableConfig { ) private long followerReadLocalLeaseTimeMs; + @Config(key = "ozone.om.block.write.sort.datanodes.enabled", + defaultValue = "false", + type = ConfigType.BOOLEAN, + tags = {ConfigTag.OM, ConfigTag.PERFORMANCE}, + description = "If true, OM sorts the streaming-write pipeline (nearest " + + "datanode first) locally using its cached cluster topology, instead " + + "of asking SCM to sort on every allocateBlock. Defaults to false so " + + "SCM performs the sort. Enable this to offload the sort from SCM " + + "when multiple OM services share a single SCM service." + ) + private boolean sortDatanodesForWriteEnabled; + public long getRatisBasedFinalizationTimeout() { return ratisBasedFinalizationTimeout; } @@ -224,6 +236,10 @@ public void setAllowLeaderSkipLinearizableRead(boolean newValue) { allowLeaderSkipLinearizableRead = newValue; } + public boolean isSortDatanodesForWriteEnabled() { + return sortDatanodesForWriteEnabled; + } + public boolean isFollowerReadLocalLeaseEnabled() { return followerReadLocalLeaseEnabled; }
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java index 8da2be2..463b9de 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java
@@ -397,6 +397,31 @@ public Builder toBuilder() { } /** + * Returns a copy of this bucket with operational properties taken from + * {@code source}. Link identity fields (volume, name, owner, source path, + * ACLs, timestamps, object/update IDs) are unchanged. + * + * <p>When adding new operational bucket fields, update this method if they + * should be resolved from a link's source bucket. + */ + public OmBucketInfo withOperationalPropertiesFrom(OmBucketInfo source) { + return toBuilder() + .setDefaultReplicationConfig(source.getDefaultReplicationConfig()) + .setIsVersionEnabled(source.getIsVersionEnabled()) + .setStorageType(source.getStorageType()) + .setQuotaInBytes(source.getQuotaInBytes()) + .setQuotaInNamespace(source.getQuotaInNamespace()) + .setUsedBytes(source.getUsedBytes()) + .setUsedNamespace(source.getUsedNamespace()) + .setSnapshotUsedBytes(source.getSnapshotUsedBytes()) + .setSnapshotUsedNamespace(source.getSnapshotUsedNamespace()) + .addAllMetadata(source.getMetadata()) + .setBucketLayout(source.getBucketLayout()) + .setTags(source.getTags()) + .build(); + } + + /** * Builder for OmBucketInfo. */ public static class Builder extends WithObjectID.Builder<OmBucketInfo> {
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartKeyInfo.java index cfa92fa..7e170d5 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartKeyInfo.java
@@ -41,8 +41,11 @@ * upload part information of the key. */ public final class OmMultipartKeyInfo extends WithObjectID implements CopyObject<OmMultipartKeyInfo> { - public static final byte LEGACY_SCHEMA_VERSION = 0; - public static final byte SPLIT_PARTS_TABLE_SCHEMA_VERSION = 1; + // This stores the schema version of the multipart key. + // 0 - Legacy Schema -> Uses the same table to store the multipart part info + // 1 - New Schema -> Uses a separate table to store the multipart part info + public static final int LEGACY_SCHEMA_VERSION = 0; + public static final int SPLIT_PARTS_TABLE_SCHEMA_VERSION = 1; private static final Codec<OmMultipartKeyInfo> CODEC = new DelegatedCodec<>( Proto2Codec.get(MultipartKeyInfo.getDefaultInstance()), @@ -87,10 +90,7 @@ public final class OmMultipartKeyInfo extends WithObjectID implements CopyObject */ private final long parentID; - // This stores the schema version of the multipart key. - // 0 - Legacy Schema -> Uses the same table to store the multipart part info - // 1 - New Schema -> Uses a separate table to store the multipart part info - private final byte schemaVersion; + private final int schemaVersion; public static Codec<OmMultipartKeyInfo> getCodec() { return CODEC; @@ -275,7 +275,7 @@ public ReplicationConfig getReplicationConfig() { return replicationConfig; } - public byte getSchemaVersion() { + public int getSchemaVersion() { return schemaVersion; } @@ -297,7 +297,7 @@ public static class Builder extends WithObjectID.Builder<OmMultipartKeyInfo> { private final AclListBuilder acls; private final TreeMap<Integer, PartKeyInfo> partKeyInfoList; private long parentID; - private byte schemaVersion; + private int schemaVersion; public Builder() { this.acls = AclListBuilder.empty(); @@ -410,8 +410,8 @@ public Builder setParentID(long parentObjId) { return this; } - public Builder setSchemaVersion(byte schemaVersion) { - this.schemaVersion = schemaVersion; + public Builder setSchemaVersion(int schemaVersion) { + this.schemaVersion = validateAndConvertSchemaVersion(schemaVersion); return this; } @@ -457,7 +457,7 @@ public static Builder builderFromProto( .setObjectID(multipartKeyInfo.getObjectID()) .setUpdateID(multipartKeyInfo.getUpdateID()) .setParentID(multipartKeyInfo.getParentID()) - .setSchemaVersion((byte) multipartKeyInfo.getSchemaVersion()); + .setSchemaVersion(validateAndConvertSchemaVersion(multipartKeyInfo.getSchemaVersion())); } /** @@ -516,6 +516,14 @@ public MultipartKeyInfo getProto() { return builder.build(); } + private static int validateAndConvertSchemaVersion(int schemaVersion) { + if (schemaVersion != LEGACY_SCHEMA_VERSION && schemaVersion != SPLIT_PARTS_TABLE_SCHEMA_VERSION) { + throw new IllegalArgumentException("Unsupported schemaVersion: " + + schemaVersion + ". Expected one of [0, 1]."); + } + return schemaVersion; + } + @Override public String getObjectInfo() { return getProto().toString();
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartPartInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartPartInfo.java index 19d8aa9..9a00f08 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartPartInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartPartInfo.java
@@ -20,7 +20,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Objects; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileEncryptionInfo; @@ -70,6 +69,10 @@ private OmMultipartPartInfo(Builder b) { if (b.partNumber <= 0) { throw new IllegalArgumentException("partNumber is required and > 0"); } + // An ETag is MANDATORY for every multipart part stored in the split parts-table schema, + // for ALL clients. The S3 gateway already computes the MD5 ETag, any other client must also supply one. + // This mirrors the AWS S3 contract where UploadPart always yields an ETag that CompleteMultipartUpload requires, + // and lets the ETag be used for part validation and returned by listParts. if (StringUtils.isBlank(b.eTag)) { throw new IllegalArgumentException("eTag is required"); } @@ -158,8 +161,7 @@ public Builder setETag(String eTagValue) { return this; } - public Builder setKeyLocationInfos( - List<OmKeyLocationInfoGroup> keyLocationInfos) { + public Builder setKeyLocationInfos(List<OmKeyLocationInfoGroup> keyLocationInfos) { this.keyLocationInfos = new ArrayList<>(keyLocationInfos); return this; } @@ -179,38 +181,33 @@ public OmMultipartPartInfo build() { } } - public static OmMultipartPartInfo getFromProto( - MultipartPartInfo multipartPartInfo) { + public static OmMultipartPartInfo getFromProto(MultipartPartInfo multipartPartInfo) { validateRequiredProtoFields(multipartPartInfo); Builder builder = new Builder() .setPartName(multipartPartInfo.getPartName()) .setPartNumber(multipartPartInfo.getPartNumber()) .setDataSize(multipartPartInfo.getDataSize()) .setModificationTime(multipartPartInfo.getModificationTime()) - .setETag(multipartPartInfo.getETag()) .setKeyLocationInfos(getKeyLocationInfosFromProto(multipartPartInfo)) + .setETag(multipartPartInfo.getETag()) .setEncInfo(null); if (!multipartPartInfo.hasObjectID()) { - LOG.warn("MultipartPartInfo missing objectID for part {}", - multipartPartInfo.getPartNumber()); + LOG.warn("MultipartPartInfo missing objectID for part {}", multipartPartInfo.getPartNumber()); } builder.setObjectID(multipartPartInfo.getObjectID()); if (!multipartPartInfo.hasUpdateID()) { - LOG.warn("MultipartPartInfo missing updateID for part {}", - multipartPartInfo.getPartNumber()); + LOG.warn("MultipartPartInfo missing updateID for part {}", multipartPartInfo.getPartNumber()); } builder.setUpdateID(multipartPartInfo.getUpdateID()); if (multipartPartInfo.hasFileEncryptionInfo()) { - builder.setEncInfo( - OMPBHelper.convert(multipartPartInfo.getFileEncryptionInfo())); + builder.setEncInfo(OMPBHelper.convert(multipartPartInfo.getFileEncryptionInfo())); } if (multipartPartInfo.hasFileChecksum()) { - builder.setFileChecksum( - OMPBHelper.convert(multipartPartInfo.getFileChecksum())); + builder.setFileChecksum(OMPBHelper.convert(multipartPartInfo.getFileChecksum())); } return builder.build(); @@ -232,6 +229,10 @@ public MultipartPartInfo getProto() { if (keyLocationInfos == null || keyLocationInfos.isEmpty()) { throw new IllegalArgumentException("keyLocationList is required"); } + if (StringUtils.isBlank(eTag)) { + throw new IllegalArgumentException("eTag is required"); + } + MultipartPartInfo.Builder builder = MultipartPartInfo.newBuilder() .setPartName(partName) .setPartNumber(partNumber) @@ -240,7 +241,7 @@ public MultipartPartInfo getProto() { .setModificationTime(modificationTime) .setObjectID(objectID) .setUpdateID(updateID) - .setETag(Objects.requireNonNull(eTag, "eTag is required")); + .setETag(eTag); if (encInfo != null) { builder.setFileEncryptionInfo(OMPBHelper.convert(encInfo)); @@ -355,6 +356,7 @@ private static void validateRequiredProtoFields(MultipartPartInfo partInfo) { if (!partInfo.hasPartNumber()) { throw new IllegalArgumentException("MultipartPartInfo missing partNumber"); } + if (!partInfo.hasETag() || StringUtils.isBlank(partInfo.getETag())) { throw new IllegalArgumentException("MultipartPartInfo missing eTag"); }
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmBucketInfo.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmBucketInfo.java index 857103a..5f816ff 100644 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmBucketInfo.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmBucketInfo.java
@@ -98,6 +98,56 @@ public void testClone() { } @Test + public void testWithOperationalPropertiesFromPreservesLinkIdentity() { + ECReplicationConfig sourceReplication = new ECReplicationConfig(3, 2); + OmBucketInfo source = OmBucketInfo.newBuilder() + .setVolumeName("vol1") + .setBucketName("source") + .setBucketLayout(BucketLayout.OBJECT_STORE) + .setStorageType(StorageType.SSD) + .setIsVersionEnabled(true) + .setQuotaInBytes(1000) + .setQuotaInNamespace(10) + .setUsedBytes(500) + .setUsedNamespace(5) + .setDefaultReplicationConfig(new DefaultReplicationConfig(sourceReplication)) + .addAllMetadata(Collections.singletonMap("sourceKey", "sourceValue")) + .build(); + + OmBucketInfo link = OmBucketInfo.newBuilder() + .setVolumeName("vol1") + .setBucketName("link") + .setSourceVolume("vol1") + .setSourceBucket("source") + .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED) + .setCreationTime(123L) + .setModificationTime(456L) + .addAllMetadata(Collections.singletonMap("linkKey", "linkValue")) + .build(); + + OmBucketInfo resolvedLink = link.withOperationalPropertiesFrom(source); + + assertEquals("link", resolvedLink.getBucketName()); + assertEquals("vol1", resolvedLink.getVolumeName()); + assertEquals("vol1", resolvedLink.getSourceVolume()); + assertEquals("source", resolvedLink.getSourceBucket()); + assertEquals(123L, resolvedLink.getCreationTime()); + assertEquals(456L, resolvedLink.getModificationTime()); + + assertEquals(BucketLayout.OBJECT_STORE, resolvedLink.getBucketLayout()); + assertEquals(StorageType.SSD, resolvedLink.getStorageType()); + assertTrue(resolvedLink.getIsVersionEnabled()); + assertEquals(1000, resolvedLink.getQuotaInBytes()); + assertEquals(10, resolvedLink.getQuotaInNamespace()); + assertEquals(500, resolvedLink.getUsedBytes()); + assertEquals(5, resolvedLink.getUsedNamespace()); + assertEquals(sourceReplication, + resolvedLink.getDefaultReplicationConfig().getReplicationConfig()); + assertEquals("sourceValue", resolvedLink.getMetadata().get("sourceKey")); + assertEquals("linkValue", resolvedLink.getMetadata().get("linkKey")); + } + + @Test public void getProtobufMessageEC() { OmBucketInfo omBucketInfo = OmBucketInfo.newBuilder().setBucketName("bucket").setVolumeName("vol1")
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmMultipartKeyInfo.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmMultipartKeyInfo.java index 3a5658d..4033d94 100644 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmMultipartKeyInfo.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmMultipartKeyInfo.java
@@ -120,7 +120,7 @@ public void distinctListOfParts() { @Test public void addPartKeyInfoRejectsSchemaVersionOne() { OmMultipartKeyInfo subject = createSubject() - .setSchemaVersion((byte) 1) + .setSchemaVersion(1) .build(); assertThrows(IllegalStateException.class, @@ -128,7 +128,7 @@ public void addPartKeyInfoRejectsSchemaVersionOne() { } @Test - public void getProtoRejectsLegacyPartListForSchemaVersionOne() { + public void getProtoRejectsPartListForSchemaVersionOne() { PartKeyInfo part = createPart(createKeyInfo()).build(); TreeMap<Integer, PartKeyInfo> legacyMap = new TreeMap<>(); legacyMap.put(part.getPartNumber(), part); @@ -136,7 +136,7 @@ public void getProtoRejectsLegacyPartListForSchemaVersionOne() { OmMultipartKeyInfo subject = new OmMultipartKeyInfo.Builder() .setUploadID(UUID.randomUUID().toString()) .setCreationTime(Time.now()) - .setSchemaVersion((byte) 1) + .setSchemaVersion(1) .setReplicationConfig(StandaloneReplicationConfig.getInstance( HddsProtos.ReplicationFactor.ONE)) .setPartKeyInfoList(legacyMap) @@ -145,6 +145,22 @@ public void getProtoRejectsLegacyPartListForSchemaVersionOne() { assertThrows(IllegalStateException.class, subject::getProto); } + @Test + public void builderFromProtoRejectsUnsupportedSchemaVersion() { + OmMultipartKeyInfo subject = createSubject() + .setReplicationConfig(StandaloneReplicationConfig.getInstance( + HddsProtos.ReplicationFactor.ONE)) + .build(); + + OzoneManagerProtocolProtos.MultipartKeyInfo invalidProto = subject.getProto() + .toBuilder() + .setSchemaVersion(256) + .build(); + + assertThrows(IllegalArgumentException.class, + () -> OmMultipartKeyInfo.getFromProto(invalidProto)); + } + private static OmMultipartKeyInfo.Builder createSubject() { return new OmMultipartKeyInfo.Builder() .setUploadID(UUID.randomUUID().toString())
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/bucketcreate.robot b/hadoop-ozone/dist/src/main/smoketest/s3/bucketcreate.robot index 4ca7a1d..3b86e61 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/bucketcreate.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/bucketcreate.robot
@@ -35,7 +35,7 @@ Create bucket which already exists ${bucket} = Create bucket ${result} = Execute AWSS3APICli and checkrc create-bucket --bucket ${bucket} 255 - Should contain ${result} BucketAlreadyExists + Should contain ${result} BucketAlreadyOwnedByYou Create bucket with invalid bucket name ${randStr} = Generate Ozone String
diff --git a/hadoop-ozone/dist/src/shell/ozone/ozone b/hadoop-ozone/dist/src/shell/ozone/ozone index ecddd4f..8592ef9 100755 --- a/hadoop-ozone/dist/src/shell/ozone/ozone +++ b/hadoop-ozone/dist/src/shell/ozone/ozone
@@ -66,6 +66,7 @@ ozone_add_subcommand "s3" client "command line interface for s3 related operations" ozone_add_subcommand "tenant" client "command line interface for multi-tenant related operations" ozone_add_subcommand "insight" client "tool to get runtime operation information" + ozone_add_subcommand "local" client "run a single-node local ozone cluster" ozone_add_subcommand "version" client "print the version" ozone_add_subcommand "dtutil" client "operations related to delegation tokens" ozone_add_subcommand "interactive" client "interactive shell for ozone commands" @@ -208,6 +209,10 @@ OZONE_CLASSNAME=org.apache.hadoop.ozone.insight.Insight OZONE_RUN_ARTIFACT_NAME="ozone-insight" ;; + local) + OZONE_CLASSNAME=org.apache.hadoop.ozone.local.OzoneLocal + OZONE_RUN_ARTIFACT_NAME="ozone-tools" + ;; version) OZONE_CLASSNAME=org.apache.hadoop.ozone.util.OzoneVersionInfo OZONE_RUN_ARTIFACT_NAME="ozone-tools"
diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3KeyGenerator.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3KeyGenerator.java index 0c43923..05bb751 100644 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3KeyGenerator.java +++ b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/S3KeyGenerator.java
@@ -17,7 +17,6 @@ package org.apache.hadoop.ozone.freon; -import static com.amazonaws.services.s3.internal.SkipMd5CheckStrategy.DISABLE_PUT_OBJECT_MD5_VALIDATION_PROPERTY; import static org.apache.hadoop.ozone.OzoneConsts.OM_MULTIPART_MIN_SIZE; import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; @@ -101,7 +100,7 @@ public Void call() throws Exception { timer = getMetrics().timer("key-create"); - System.setProperty(DISABLE_PUT_OBJECT_MD5_VALIDATION_PROPERTY, "true"); + System.setProperty("com.amazonaws.services.s3.disablePutObjectMD5Validation", "true"); runTests(this::createKey); return null;
diff --git a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/AbstractTestStorageDistributionEndpoint.java b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/AbstractTestStorageDistributionEndpoint.java index ee63904..8ea2d8c 100644 --- a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/AbstractTestStorageDistributionEndpoint.java +++ b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/AbstractTestStorageDistributionEndpoint.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; @@ -207,6 +208,7 @@ protected void createOpenKeysAndMultipartKeys(String volumeName, .createMultipartKey(volumeName, bucketName, "mpukey1", 100L, 1, multipartInfo.getUploadID()); partStream.write(new byte[100]); + partStream.getMetadata().put(OzoneConsts.ETAG, "mpukey1-part1-etag"); partStream.close(); }
diff --git a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryMemoryLeak.java b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryMemoryLeak.java index 34ba542..58bd673 100644 --- a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryMemoryLeak.java +++ b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryMemoryLeak.java
@@ -35,9 +35,9 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -149,7 +149,7 @@ public static void init() throws Exception { client = cluster.newClient(); // Create FSO bucket for testing - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client, + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client, BucketLayout.FILE_SYSTEM_OPTIMIZED); String volumeName = bucket.getVolumeName(); String bucketName = bucket.getName();
diff --git a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconAndAdminContainerCLI.java b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconAndAdminContainerCLI.java index 19af93b..3d3bd86 100644 --- a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconAndAdminContainerCLI.java +++ b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconAndAdminContainerCLI.java
@@ -67,9 +67,9 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.container.TestHelper; @@ -177,7 +177,7 @@ static void init() throws Exception { String volumeName = "vol1"; String bucketName = "bucket1"; - ozoneBucket = TestDataUtil.createVolumeAndBucket( + ozoneBucket = DataTestUtil.createVolumeAndBucket( client, volumeName, bucketName, BucketLayout.FILE_SYSTEM_OPTIMIZED); String keyNameR3 = "key1"; @@ -432,7 +432,7 @@ private static OmKeyInfo createTestKey(String keyName, ReplicationConfig replicationConfig) throws IOException { byte[] textBytes = "Testing".getBytes(UTF_8); - TestDataUtil.createKey(ozoneBucket, keyName, replicationConfig, textBytes); + DataTestUtil.createKey(ozoneBucket, keyName, replicationConfig, textBytes); OmKeyArgs keyArgs = new OmKeyArgs.Builder() .setVolumeName(ozoneBucket.getVolumeName())
diff --git a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconInsightsForDeletedDirectories.java b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconInsightsForDeletedDirectories.java index d7a2dea..fc83693 100644 --- a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconInsightsForDeletedDirectories.java +++ b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconInsightsForDeletedDirectories.java
@@ -50,9 +50,9 @@ import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -159,7 +159,7 @@ public void cleanup() throws IOException { @MethodSource("replicationConfigs") public void testGetDeletedDirectoryInfo(ReplicationConfig replicationConfig) throws Exception { - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client, BucketLayout.FILE_SYSTEM_OPTIMIZED, + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client, BucketLayout.FILE_SYSTEM_OPTIMIZED, new DefaultReplicationConfig(replicationConfig)); String rootPath = String.format("%s://%s.%s/", OzoneConsts.OZONE_URI_SCHEME, bucket.getName(), bucket.getVolumeName()); @@ -283,7 +283,7 @@ public void testGetDeletedDirectoryInfo(ReplicationConfig replicationConfig) @MethodSource("replicationConfigs") public void testGetDeletedDirectoryInfoForNestedDirectories(ReplicationConfig replicationConfig) throws Exception { - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client, BucketLayout.FILE_SYSTEM_OPTIMIZED, + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client, BucketLayout.FILE_SYSTEM_OPTIMIZED, new DefaultReplicationConfig(replicationConfig)); String rootPath = String.format("%s://%s.%s/", OzoneConsts.OZONE_URI_SCHEME, bucket.getName(), bucket.getVolumeName()); @@ -395,7 +395,7 @@ public void testGetDeletedDirectoryInfoForNestedDirectories(ReplicationConfig re @MethodSource("replicationConfigs") public void testGetDeletedDirectoryInfoWithMultipleSubdirectories(ReplicationConfig replicationConfig) throws Exception { - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client, BucketLayout.FILE_SYSTEM_OPTIMIZED, + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client, BucketLayout.FILE_SYSTEM_OPTIMIZED, new DefaultReplicationConfig(replicationConfig)); String rootPath = String.format("%s://%s.%s/", OzoneConsts.OZONE_URI_SCHEME, bucket.getName(), bucket.getVolumeName());
diff --git a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java index dfdc3de..8896f74 100644 --- a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java +++ b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java
@@ -41,6 +41,7 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.hdds.JsonTestUtils; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; @@ -52,9 +53,11 @@ import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.recon.metrics.OzoneManagerSyncMetrics; import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; import org.apache.http.HttpEntity; @@ -74,6 +77,8 @@ * Test Ozone Recon. */ public class TestReconWithOzoneManager { + private static final AtomicLong OBJECT_ID_SEQUENCE = new AtomicLong(); + private static MiniOzoneCluster cluster = null; private static OMMetadataManager metadataManager; private static CloseableHttpClient httpClient; @@ -393,6 +398,93 @@ private long getReconTaskAttributeFromJson(String taskStatusResponse, * Helper function to add voli/bucketi/keyi to containeri to OM Metadata. * For test purpose each container will have only one key. */ + @Test + public void testManualOMDBRebuild() throws Exception { + // 1. Stop Recon + recon.stop(); + + // 2. Write keys to OM + addKeys(20, 25); + long omLatestSeqNumber = ((RDBStore) metadataManager.getStore()) + .getDb().getLatestSequenceNumber(); + java.io.File omDbDir = metadataManager.getStore().getDbLocation(); + + // 3. Stop OM (flush to disk) + cluster.getOzoneManager().stop(); + + // 4. Copy OM DB into Recon's OM snapshot dir. The dir and the "om.snapshot.db_" prefix must + // match what ReconOmMetadataManagerImpl#start looks up via ReconUtils#getLastKnownDB, otherwise + // Recon will not load the copied DB on restart. + java.io.File reconOmSnapshotDir = new ReconUtils() + .getReconDbDir(cluster.getConf(), ReconServerConfigKeys.OZONE_RECON_OM_SNAPSHOT_DB_DIR); + java.io.File reconOmDbDir = new java.io.File(reconOmSnapshotDir, + ReconConstants.RECON_OM_SNAPSHOT_DB + "_" + System.currentTimeMillis()); + org.apache.commons.io.FileUtils.deleteDirectory(reconOmDbDir); + org.apache.commons.io.FileUtils.copyDirectory(omDbDir, reconOmDbDir); + + // 5. Restart Recon and confirm it loaded the OM DB copy we placed above. Shutting OM down + // writes its own trailing records, so the copy is at or ahead of the sequence number we + // captured while OM was up. + recon.start(cluster.getConf()); + OzoneManagerServiceProviderImpl reconImpl = (OzoneManagerServiceProviderImpl) + recon.getReconServer().getOzoneManagerServiceProvider(); + long reconLoadedSeqNumber = ((RDBStore) reconImpl.getOMMetadataManagerInstance().getStore()) + .getDb().getLatestSequenceNumber(); + assertThat(reconLoadedSeqNumber).isGreaterThanOrEqualTo(omLatestSeqNumber); + + // 6. POST reinit + String triggerUrl = "http://" + cluster.getConf().get(OZONE_RECON_HTTP_ADDRESS_KEY) + + "/api/v1/triggerdbsync/om/reinit"; + org.apache.http.client.methods.HttpPost httpPost = new org.apache.http.client.methods.HttpPost(triggerUrl); + HttpResponse response = httpClient.execute(httpPost); + assertEquals(202, response.getStatusLine().getStatusCode()); + + // 7. Wait for reprocess to succeed (REPROCESS_STAGING seq is set by reInitializeTasks) + GenericTestUtils.waitFor(() -> { + try { + String taskStatusResponse = makeHttpCall(taskStatusURL); + long reconLatestSeqNumber = getReconTaskAttributeFromJson( + taskStatusResponse, + "REPROCESS_STAGING", + "lastUpdatedSeqNumber"); + return reconLatestSeqNumber == reconLoadedSeqNumber; + } catch (Exception e) { + return false; + } + }, 1000, 30000); + + // 8. Start OM + cluster.getOzoneManager().restart(); + // restart() rebuilds the OM metadata manager, so re-fetch the live handle. + refreshOmMetadataManager(); + + // 9. Write more keys and verify delta resumes + addKeys(25, 30); + long newOmLatestSeqNumber = ((RDBStore) cluster.getOzoneManager().getMetadataManager().getStore()) + .getDb().getLatestSequenceNumber(); + + OzoneManagerServiceProviderImpl newImpl = (OzoneManagerServiceProviderImpl) + recon.getReconServer().getOzoneManagerServiceProvider(); + newImpl.syncDataFromOM(); + + GenericTestUtils.waitFor(() -> { + try { + String taskStatusResponse = makeHttpCall(taskStatusURL); + long reconLatestSeqNumber = getReconTaskAttributeFromJson( + taskStatusResponse, + OmSnapshotRequest.name(), + "lastUpdatedSeqNumber"); + return reconLatestSeqNumber == newOmLatestSeqNumber; + } catch (Exception e) { + return false; + } + }, 1000, 30000); + } + + private static void refreshOmMetadataManager() { + metadataManager = cluster.getOzoneManager().getMetadataManager(); + } + private void addKeys(int start, int end) throws Exception { for (int i = start; i < end; i++) { Pipeline pipeline = HddsTestUtils.getRandomPipeline(); @@ -421,6 +513,30 @@ private static void writeDataToOm(String key, String bucket, String volume, omKeyLocationInfoGroupList) throws IOException { + // Recon's full reprocess resolves the parent bucket of every key it reads, so the volume + // and bucket rows have to exist next to the key entry. + String volumeKey = metadataManager.getVolumeKey(volume); + if (metadataManager.getVolumeTable().get(volumeKey) == null) { + metadataManager.getVolumeTable().put(volumeKey, + OmVolumeArgs.newBuilder() + .setVolume(volume) + .setAdminName("TestUser") + .setOwnerName("TestUser") + .setObjectID(OBJECT_ID_SEQUENCE.incrementAndGet()) + .build()); + } + + String bucketKey = metadataManager.getBucketKey(volume, bucket); + if (metadataManager.getBucketTable().get(bucketKey) == null) { + metadataManager.getBucketTable().put(bucketKey, + OmBucketInfo.newBuilder() + .setVolumeName(volume) + .setBucketName(bucket) + .setBucketLayout(getBucketLayout()) + .setObjectID(OBJECT_ID_SEQUENCE.incrementAndGet()) + .build()); + } + String omKey = metadataManager.getOzoneKey(volume, bucket, key);
diff --git a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java index 5163813..bac55b7 100644 --- a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java +++ b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java
@@ -30,8 +30,8 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -92,7 +92,7 @@ private void writeKeys(String vol, String bucket, String key) String keyString = UUID.randomUUID().toString(); byte[] data = ContainerTestHelper.getFixedLengthString( keyString, 100).getBytes(UTF_8); - TestDataUtil.createKey(ozoneBucket, key, data); + DataTestUtil.createKey(ozoneBucket, key, data); } @Test
diff --git a/hadoop-ozone/integration-test-s3/pom.xml b/hadoop-ozone/integration-test-s3/pom.xml index 75c3e37..ab228d7 100644 --- a/hadoop-ozone/integration-test-s3/pom.xml +++ b/hadoop-ozone/integration-test-s3/pom.xml
@@ -27,7 +27,7 @@ <properties> <!-- move to root POM when needed in any other module --> - <aws-java-sdk2.version>2.47.5</aws-java-sdk2.version> + <aws-java-sdk2.version>2.48.3</aws-java-sdk2.version> </properties> <dependencyManagement>
diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index 514834d..fc634a4 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
@@ -129,6 +129,7 @@ import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -254,6 +255,35 @@ public void testCreateBucket() { assertTrue(isBucketEmpty(b)); } + /** + * s3-tests: test_bucket_create_exists. + */ + @Test + public void testCreateBucketAlreadyOwnedByYou() { + final String bucketName = getBucketName("owned-by-you"); + s3Client.createBucket(bucketName); + + AmazonServiceException ase = assertThrows(AmazonServiceException.class, + () -> s3Client.createBucket(bucketName)); + assertEquals(409, ase.getStatusCode()); + assertEquals(S3ErrorTable.BUCKET_ALREADY_OWNED_BY_YOU.getCode(), ase.getErrorCode()); + } + + @Test + public void testCreateBucketAlreadyExistsDifferentOwner() throws IOException { + final String bucketName = getBucketName("other-owner"); + final String otherOwner = "other-s3-owner"; + try (OzoneClient ozoneClient = cluster.newClient()) { + ozoneClient.getObjectStore().getS3Volume().createBucket(bucketName, + BucketArgs.newBuilder().setOwner(otherOwner).build()); + } + + AmazonServiceException ase = assertThrows(AmazonServiceException.class, + () -> s3Client.createBucket(bucketName)); + assertEquals(409, ase.getStatusCode()); + assertEquals(S3ErrorTable.BUCKET_ALREADY_EXISTS.getCode(), ase.getErrorCode()); + } + @Test public void testBucketACLOperations() { // TODO HDDS-11738: Uncomment assertions when bucket S3 ACL logic has been fixed @@ -1338,6 +1368,26 @@ public void testGetObject() throws Exception { } } + /** + * Adapted from ceph s3-tests test_object_read_unreadable. + */ + @Test + public void testGetObjectUnreadableKey() { + final String bucketName = getBucketName(); + s3Client.createBucket(bucketName); + + String unreadableKey = new String(new byte[] {(byte) 0xae, (byte) 0x8a, '-'}, + StandardCharsets.ISO_8859_1); + + AmazonServiceException ase = assertThrows(AmazonServiceException.class, + () -> s3Client.getObject(bucketName, unreadableKey)); + + assertEquals(ErrorType.Client, ase.getErrorType()); + assertEquals(400, ase.getStatusCode()); + assertEquals(S3ErrorTable.INVALID_URI.getCode(), ase.getErrorCode()); + assertEquals(S3ErrorTable.INVALID_URI.getErrorMessage(), ase.getErrorMessage()); + } + static Stream<Arguments> onlyTagKeyCasesV1() { Map<String, String> fooBarEmptyBar = new HashMap<>(); fooBarEmptyBar.put("foo", "bar");
diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index 34cccac..2fe7857 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
@@ -233,6 +233,37 @@ void closeClient() { } } + /** + * s3-tests: test_bucket_create_exists. + */ + @Test + public void testCreateBucketAlreadyOwnedByYou() { + final String bucketName = getBucketName("owned-by-you"); + s3Client.createBucket(b -> b.bucket(bucketName)); + + S3Exception exception = assertThrows(S3Exception.class, + () -> s3Client.createBucket(b -> b.bucket(bucketName))); + assertEquals(409, exception.statusCode()); + assertEquals(S3ErrorTable.BUCKET_ALREADY_OWNED_BY_YOU.getCode(), + exception.awsErrorDetails().errorCode()); + } + + @Test + public void testCreateBucketAlreadyExistsDifferentOwner() throws IOException { + final String bucketName = getBucketName("other-owner"); + final String otherOwner = "other-s3-owner"; + try (OzoneClient ozoneClient = cluster.newClient()) { + ozoneClient.getObjectStore().getS3Volume().createBucket(bucketName, + BucketArgs.newBuilder().setOwner(otherOwner).build()); + } + + S3Exception exception = assertThrows(S3Exception.class, + () -> s3Client.createBucket(b -> b.bucket(bucketName))); + assertEquals(409, exception.statusCode()); + assertEquals(S3ErrorTable.BUCKET_ALREADY_EXISTS.getCode(), + exception.awsErrorDetails().errorCode()); + } + @Test public void testPutObject() { final String bucketName = getBucketName(); @@ -588,6 +619,25 @@ public void testGetObjectIfModifiedSinceReturnsNotModified() { assertEquals(304, exception.statusCode()); } + /** + * Adapted from ceph s3-tests test_object_read_unreadable. + */ + @Test + public void testGetObjectUnreadableKey() { + final String bucketName = getBucketName(); + s3Client.createBucket(b -> b.bucket(bucketName)); + + String unreadableKey = new String(new byte[] {(byte) 0xae, (byte) 0x8a, '-'}, + StandardCharsets.ISO_8859_1); + + S3Exception exception = assertThrows(S3Exception.class, + () -> s3Client.getObjectAsBytes(b -> b.bucket(bucketName).key(unreadableKey))); + + assertEquals(400, exception.statusCode()); + assertEquals(S3ErrorTable.INVALID_URI.getCode(), exception.awsErrorDetails().errorCode()); + assertEquals(S3ErrorTable.INVALID_URI.getErrorMessage(), exception.awsErrorDetails().errorMessage()); + } + @Test public void testHeadObjectIfMatch() { final String bucketName = getBucketName();
diff --git a/hadoop-ozone/integration-test/dev-support/findbugsExcludeFile.xml b/hadoop-ozone/integration-test/dev-support/findbugsExcludeFile.xml index 55abc26..bbc0b6b 100644 --- a/hadoop-ozone/integration-test/dev-support/findbugsExcludeFile.xml +++ b/hadoop-ozone/integration-test/dev-support/findbugsExcludeFile.xml
@@ -16,4 +16,16 @@ limitations under the License. --> <FindBugsFilter> + <Match> + <Class name="org.apache.hadoop.hdds.scm.TestRackAwarePlacement$WithRacksAndHosts"/> + <Bug pattern="SIC_INNER_SHOULD_BE_STATIC"/> + </Match> + <Match> + <Class name="org.apache.hadoop.hdds.scm.TestRackAwarePlacement$WithRacksOnly"/> + <Bug pattern="SIC_INNER_SHOULD_BE_STATIC"/> + </Match> + <Match> + <Class name="org.apache.hadoop.hdds.scm.TestRackAwarePlacement$WithHostsOnly"/> + <Bug pattern="SIC_INNER_SHOULD_BE_STATIC"/> + </Match> </FindBugsFilter>
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java index 638bb93..302c377 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java
@@ -92,10 +92,10 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; @@ -198,7 +198,7 @@ void init() throws Exception { writeClient = client.getObjectStore() .getClientProxy().getOzoneManagerClient(); // create a volume and a bucket to be used by OzoneFileSystem - ozoneBucket = TestDataUtil.createVolumeAndBucket(client, bucketLayout); + ozoneBucket = DataTestUtil.createVolumeAndBucket(client, bucketLayout); volumeName = ozoneBucket.getVolumeName(); bucketName = ozoneBucket.getName(); @@ -368,7 +368,7 @@ public void testMakeDirsWithAnFakeDirectory() throws Exception { String fakeGrandpaKey = "dir1"; String fakeParentKey = fakeGrandpaKey + "/dir2"; String fullKeyName = fakeParentKey + "/key1"; - TestDataUtil.createKey(ozoneBucket, fullKeyName, new byte[0]); + DataTestUtil.createKey(ozoneBucket, fullKeyName, new byte[0]); // /dir1/dir2 should not exist assertFalse(fs.exists(new Path(fakeParentKey))); @@ -416,10 +416,10 @@ void testContentSummaryErasureCodingPolicy() throws Exception { fs.mkdirs(parentDir); String ratisRelKey = "ec-policy-mixed-o3fs/" + ratisKey; String ecRelKey = "ec-policy-mixed-o3fs/" + ecKey; - TestDataUtil.createKey(ozoneBucket, ratisRelKey, + DataTestUtil.createKey(ozoneBucket, ratisRelKey, RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), new byte[]{0}); - TestDataUtil.createKey(ozoneBucket, ecRelKey, ecConfig, + DataTestUtil.createKey(ozoneBucket, ecRelKey, ecConfig, new byte[]{0}); try { @@ -792,7 +792,7 @@ public void testListStatusOnKeyNameContainDelimiter() throws Exception { * the "/dir1", "/dir1/dir2/" are fake directory * */ String keyName = "dir1/dir2/key1"; - TestDataUtil.createKey(ozoneBucket, keyName, new byte[0]); + DataTestUtil.createKey(ozoneBucket, keyName, new byte[0]); FileStatus[] fileStatuses; fileStatuses = fs.listStatus(ROOT, EXCLUDE_TRASH); @@ -1119,7 +1119,7 @@ public void testRenameContainDelimiterFile() throws Exception { String fakeParentKey = fakeGrandpaKey + "/dir2"; String sourceKeyName = fakeParentKey + "/key1"; String targetKeyName = fakeParentKey + "/key2"; - TestDataUtil.createKey(ozoneBucket, sourceKeyName, new byte[0]); + DataTestUtil.createKey(ozoneBucket, sourceKeyName, new byte[0]); Path sourcePath = pathUnderFsRoot("/" + sourceKeyName); Path targetPath = pathUnderFsRoot("/" + targetKeyName); @@ -1246,7 +1246,7 @@ public void testGetTrashRoot() throws IOException { public void testCreateKeyShouldUseRefreshedBucketReplicationConfig() throws IOException { OzoneBucket bucket = - TestDataUtil.createVolumeAndBucket(client, bucketLayout); + DataTestUtil.createVolumeAndBucket(client, bucketLayout); final MockClock testClock = new MockClock(Instant.now(), ZoneOffset.UTC); String rootPath = String @@ -1552,7 +1552,7 @@ public void testProcessingDetails() throws IOException, InterruptedException { GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer.captureLogs(log); int keySize = 1024; - TestDataUtil.createKey(ozoneBucket, "key1", new byte[keySize]); + DataTestUtil.createKey(ozoneBucket, "key1", new byte[keySize]); logCapturer.stopCapturing(); String logContent = logCapturer.getOutput();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java index e479cbb..a0d0926 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java
@@ -97,12 +97,12 @@ import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OFSPath; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; @@ -196,7 +196,7 @@ void shutdown() { void createVolumeAndBucket() throws IOException { // create a volume and a bucket to be used by RootedOzoneFileSystem (OFS) OzoneBucket bucket = - TestDataUtil.createVolumeAndBucket(client, bucketLayout); + DataTestUtil.createVolumeAndBucket(client, bucketLayout); volumeName = bucket.getVolumeName(); volumePath = new Path(OZONE_URI_DELIMITER, volumeName); bucketName = bucket.getName(); @@ -304,7 +304,7 @@ void testListStatusWithIntermediateDirWithECEnabled() String key = "object-dir/object-name1"; // write some test data into bucket - TestDataUtil.createKey(objectStore.getVolume(volumeName).getBucket(bucketName), + DataTestUtil.createKey(objectStore.getVolume(volumeName).getBucket(bucketName), key, new ECReplicationConfig("RS-3-2-1024k"), RandomUtils.secure().randomBytes(1)); @@ -1251,7 +1251,7 @@ void testSymlinkList() throws Exception { // add key in source bucket final String key = "object-dir/object-name1"; - TestDataUtil.createKey(objectStore.getVolume(srcVolume).getBucket(srcBucket), + DataTestUtil.createKey(objectStore.getVolume(srcVolume).getBucket(srcBucket), key, RandomUtils.secure().randomBytes(1)); assertEquals(key, objectStore.getVolume(srcVolume) .getBucket(srcBucket).getKey(key).getName()); @@ -1299,7 +1299,7 @@ void testSymlinkPosixDelete() throws Exception { // add key to srcBucket final String key = "object-dir/object-name1"; - TestDataUtil.createKey(objectStore.getVolume(srcVolume).getBucket(srcBucket), + DataTestUtil.createKey(objectStore.getVolume(srcVolume).getBucket(srcBucket), key, RandomUtils.secure().randomBytes(1)); assertEquals(key, objectStore.getVolume(srcVolume). getBucket(srcBucket).getKey(key).getName()); @@ -1535,7 +1535,7 @@ void testGetTrashRoots() throws IOException { // Create a new volume and a new bucket OzoneBucket bucket3 = - TestDataUtil.createVolumeAndBucket(client, bucketLayout); + DataTestUtil.createVolumeAndBucket(client, bucketLayout); OzoneVolume volume3 = objectStore.getVolume(bucket3.getVolumeName()); // Need to setOwner to current test user so it has permission to list vols volume3.setOwner(username); @@ -1596,7 +1596,7 @@ void testTrash() throws Exception { } // create second bucket and write a key in it. OzoneBucket bucket2 = - TestDataUtil.createVolumeAndBucket(client, bucketLayout); + DataTestUtil.createVolumeAndBucket(client, bucketLayout); String volumeName2 = bucket2.getVolumeName(); Path volumePath2 = new Path(OZONE_URI_DELIMITER, volumeName2); String bucketName2 = bucket2.getName(); @@ -1790,7 +1790,7 @@ void testBucketDefaultsShouldNotBeInheritedToFileForNonEC() BucketArgs omBucketArgs = builder.build(); String vol = UUID.randomUUID().toString(); String buck = UUID.randomUUID().toString(); - final OzoneBucket bucket100 = TestDataUtil + final OzoneBucket bucket100 = DataTestUtil .createVolumeAndBucket(client, vol, buck, omBucketArgs); assertEquals(ReplicationType.STAND_ALONE.name(), bucket100.getReplicationConfig().getReplicationType().name()); @@ -1819,7 +1819,7 @@ void testBucketDefaultsShouldBeInheritedToFileForEC() BucketArgs omBucketArgs = builder.build(); String vol = UUID.randomUUID().toString(); String buck = UUID.randomUUID().toString(); - final OzoneBucket bucket101 = TestDataUtil + final OzoneBucket bucket101 = DataTestUtil .createVolumeAndBucket(client, vol, buck, omBucketArgs); assertEquals(ReplicationType.EC.name(), bucket101.getReplicationConfig().getReplicationType().name()); @@ -1869,7 +1869,7 @@ void testCreateAndCheckECFileDiskUsage() throws Exception { Path bucketPathTest = new Path(volPathTest, bucketName); // write some test data into bucket - TestDataUtil.createKey(objectStore.getVolume(volumeName). + DataTestUtil.createKey(objectStore.getVolume(volumeName). getBucket(bucketName), key, new ECReplicationConfig("RS-3-2-1024k"), RandomUtils.secure().randomBytes(1)); // make sure the disk usage matches the expected value @@ -1897,10 +1897,10 @@ void testContentSummaryErasureCodingPolicy() throws Exception { OzoneBucket bucket = objectStore.getVolume(volumeName).getBucket(bucketName); String ratisRelKey = "ec-policy-mixed/" + ratisKey; String ecRelKey = "ec-policy-mixed/" + ecKey; - TestDataUtil.createKey(bucket, ratisRelKey, + DataTestUtil.createKey(bucket, ratisRelKey, RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), new byte[]{0}); - TestDataUtil.createKey(bucket, ecRelKey, ecConfig, + DataTestUtil.createKey(bucket, ecRelKey, ecConfig, new byte[]{0}); try { @@ -1942,7 +1942,7 @@ void testCreateAndCheckRatisFileDiskUsage() throws Exception { Path filePathTest = new Path(bucketPathTest, key); // write some test data into bucket - TestDataUtil.createKey(objectStore. + DataTestUtil.createKey(objectStore. getVolume(volumeName).getBucket(bucketName), key, RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), RandomUtils.secure().randomBytes(1)); @@ -2019,7 +2019,7 @@ private Path createAndGetBucketPath() String vol = UUID.randomUUID().toString(); String buck = UUID.randomUUID().toString(); final OzoneBucket bucket = - TestDataUtil.createVolumeAndBucket(client, vol, buck, omBucketArgs); + DataTestUtil.createVolumeAndBucket(client, vol, buck, omBucketArgs); Path volume = new Path(OZONE_URI_DELIMITER, bucket.getVolumeName()); return new Path(volume, bucket.getName()); } @@ -2028,7 +2028,7 @@ private Path createAndGetBucketPath() void testSnapshotRead() throws Exception { // Init data OzoneBucket bucket1 = - TestDataUtil.createVolumeAndBucket(client, bucketLayout); + DataTestUtil.createVolumeAndBucket(client, bucketLayout); Path volume1Path = new Path(OZONE_URI_DELIMITER, bucket1.getVolumeName()); Path bucket1Path = new Path(volume1Path, bucket1.getName()); Path file1 = new Path(bucket1Path, "key1"); @@ -2038,7 +2038,7 @@ void testSnapshotRead() throws Exception { ContractTestUtils.touch(fs, file2); OzoneBucket bucket2 = - TestDataUtil.createVolumeAndBucket(client, bucketLayout); + DataTestUtil.createVolumeAndBucket(client, bucketLayout); Path volume2Path = new Path(OZONE_URI_DELIMITER, bucket2.getVolumeName()); Path bucket2Path = new Path(volume2Path, bucket2.getName()); @@ -2071,7 +2071,7 @@ void testFileSystemDeclaresCapability() throws Throwable { @Test void testSnapshotDiff() throws Exception { OzoneBucket bucket1 = - TestDataUtil.createVolumeAndBucket(client, bucketLayout); + DataTestUtil.createVolumeAndBucket(client, bucketLayout); Path volumePath1 = new Path(OZONE_URI_DELIMITER, bucket1.getVolumeName()); Path bucketPath1 = new Path(volumePath1, bucket1.getName()); Path snap1 = fs.createSnapshot(bucketPath1); @@ -2133,7 +2133,7 @@ void testSnapshotDiff() throws Exception { @Test void testSetTimes() throws Exception { OzoneBucket bucket1 = - TestDataUtil.createVolumeAndBucket(client, bucketLayout); + DataTestUtil.createVolumeAndBucket(client, bucketLayout); Path volumePath1 = new Path(OZONE_URI_DELIMITER, bucket1.getVolumeName()); Path bucketPath1 = new Path(volumePath1, bucket1.getName()); setTimes(bucketPath1); @@ -2143,7 +2143,7 @@ void testSetTimes() throws Exception { public void testSetTimesForLinkedBucketPath() throws Exception { // Create a file OzoneBucket sourceBucket = - TestDataUtil.createVolumeAndBucket(client, bucketLayout); + DataTestUtil.createVolumeAndBucket(client, bucketLayout); Path volumePath1 = new Path(OZONE_URI_DELIMITER, sourceBucket.getVolumeName()); Path sourceBucketPath = new Path(volumePath1, sourceBucket.getName());
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java index ec5887f..bb240bb 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java
@@ -19,6 +19,8 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_RATIS_PIPELINE_LIMIT; +import static org.apache.hadoop.ozone.DataTestUtil.cleanupDeletedTable; +import static org.apache.hadoop.ozone.DataTestUtil.cleanupOpenKeyTable; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_DATASTREAM_AUTO_THRESHOLD; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLED; @@ -26,8 +28,6 @@ import static org.apache.hadoop.ozone.OzoneConsts.OZONE_ROOT; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME; -import static org.apache.hadoop.ozone.TestDataUtil.cleanupDeletedTable; -import static org.apache.hadoop.ozone.TestDataUtil.cleanupOpenKeyTable; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; @@ -93,11 +93,11 @@ import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.ClientConfigForTesting; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -208,7 +208,7 @@ public static void init() throws Exception { client = cluster.newClient(); // create a volume and a bucket to be used by OzoneFileSystem - bucket = TestDataUtil.createVolumeAndBucket(client, layout); + bucket = DataTestUtil.createVolumeAndBucket(client, layout); openKeyCleanupService = (OpenKeyCleanupService) cluster.getOzoneManager().getKeyManager().getOpenKeyCleanupService(); @@ -1077,7 +1077,7 @@ public void testECStreamCapability() throws Exception { 3, 2, ECReplicationConfig.EcCodec.RS, (int) OzoneConsts.MB))); BucketArgs omBucketArgs = builder.build(); String ecBucket = UUID.randomUUID().toString(); - TestDataUtil.createBucket(client, bucket.getVolumeName(), omBucketArgs, + DataTestUtil.createBucket(client, bucket.getVolumeName(), omBucketArgs, ecBucket); String ecUri = String.format("%s://%s.%s/", OzoneConsts.OZONE_URI_SCHEME, ecBucket, bucket.getVolumeName());
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java index 0c4f9c9..aca8b24 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java
@@ -51,9 +51,9 @@ import org.apache.hadoop.hdds.scm.storage.BufferPool; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.ClientConfigForTesting; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler; @@ -136,7 +136,7 @@ public void init() throws Exception { client = cluster.newClient(); // create a volume and a bucket to be used by OzoneFileSystem - bucket = TestDataUtil.createVolumeAndBucket(client, layout); + bucket = DataTestUtil.createVolumeAndBucket(client, layout); // Enable DEBUG level logging for relevant classes GenericTestUtils.setLogLevel(BlockManagerImpl.class, Level.DEBUG);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java index 061ced5..eda35dc 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java
@@ -59,10 +59,10 @@ import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.ClientConfigForTesting; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneTestUtils; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler; @@ -159,7 +159,7 @@ public void init() throws IOException, InterruptedException, client = cluster.newClient(); // create a volume and a bucket to be used by OzoneFileSystem - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client, layout); + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client, layout); GenericTestUtils.setLogLevel(XceiverClientGrpc.class, Level.DEBUG); @@ -260,7 +260,7 @@ public void testRecoveryWithoutHsyncHflushOnLastBlock() throws Exception { @Test public void testOBSRecoveryShouldFail() throws Exception { - OzoneBucket obsBucket = TestDataUtil.createVolumeAndBucket(client, + OzoneBucket obsBucket = DataTestUtil.createVolumeAndBucket(client, "vol2", "obs", BucketLayout.OBJECT_STORE); String obsDir = OZONE_ROOT + obsBucket.getVolumeName() + OZONE_URI_DELIMITER + obsBucket.getName(); Path obsFile = new Path(obsDir, uniqueObjectName());
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java index 0330203..86f602b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java
@@ -34,8 +34,8 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -97,7 +97,7 @@ void setUp() throws Exception { objectStore = client.getObjectStore(); rootPath = String.format("%s://%s/", OzoneConsts.OZONE_OFS_URI_SCHEME, cluster().getConf().get(OZONE_OM_ADDRESS_KEY)); - volumeName = TestDataUtil.createVolumeAndBucket(client).getVolumeName(); + volumeName = DataTestUtil.createVolumeAndBucket(client).getVolumeName(); } @AfterAll
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java index e039e14..c407316 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
@@ -33,6 +33,7 @@ import java.nio.ByteBuffer; import java.nio.file.Files; import java.util.UUID; +import java.util.stream.Stream; import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -40,11 +41,12 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ECReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -52,9 +54,10 @@ import org.apache.ozone.test.NonHATests; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; /** @@ -64,27 +67,28 @@ public abstract class TestOzoneFSInputStream implements NonHATests.TestCase { private OzoneClient client; - private FileSystem fs; private FileSystem ecFs; private Path filePath = null; private byte[] data = null; + private String uri = null; @BeforeAll void init() throws Exception { client = cluster().newClient(); // create a volume and a bucket to be used by OzoneFileSystem - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client); + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client); // Set the fs.defaultFS and start the filesystem - String uri = String.format("%s://%s.%s/", + uri = String.format("%s://%s.%s/", OzoneConsts.OZONE_URI_SCHEME, bucket.getName(), bucket.getVolumeName()); - fs = FileSystem.get(URI.create(uri), cluster().getConf()); - int fileLen = 30 * 1024 * 1024; - data = string2Bytes(RandomStringUtils.secure().nextAlphanumeric(fileLen)); - filePath = new Path("/" + RandomStringUtils.secure().nextAlphanumeric(5)); - try (FSDataOutputStream stream = fs.create(filePath)) { - stream.write(data); + try (FileSystem fs = FileSystem.get(URI.create(uri), cluster().getConf());) { + int fileLen = 30 * 1024 * 1024; + data = string2Bytes(RandomStringUtils.secure().nextAlphanumeric(fileLen)); + filePath = new Path("/" + RandomStringUtils.secure().nextAlphanumeric(5)); + try (FSDataOutputStream stream = fs.create(filePath)) { + stream.write(data); + } } // create EC bucket to be used by OzoneFileSystem @@ -97,7 +101,7 @@ void init() throws Exception { (int) OzoneConsts.MB))); BucketArgs omBucketArgs = builder.build(); String ecBucket = UUID.randomUUID().toString(); - TestDataUtil.createBucket(client, bucket.getVolumeName(), omBucketArgs, + DataTestUtil.createBucket(client, bucket.getVolumeName(), omBucketArgs, ecBucket); String ecUri = String.format("%s://%s.%s/", OzoneConsts.OZONE_URI_SCHEME, ecBucket, bucket.getVolumeName()); @@ -106,12 +110,17 @@ void init() throws Exception { @AfterAll void shutdown() { - closeQuietly(client, fs, ecFs); + closeQuietly(client, ecFs); } - @Test - public void testO3FSSingleByteRead() throws IOException { - try (FSDataInputStream inputStream = fs.open(filePath)) { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testO3FSSingleByteRead(boolean isStreamEnable) throws IOException { + OzoneConfiguration conf = cluster().getConf(); + conf.setBoolean("ozone.client.stream.readblock.enable", isStreamEnable); + + try (FileSystem fs = FileSystem.get(URI.create(uri), conf); + FSDataInputStream inputStream = fs.open(filePath)) { byte[] value = new byte[data.length]; int i = 0; while (true) { @@ -128,9 +137,13 @@ public void testO3FSSingleByteRead() throws IOException { } } - @Test - public void testByteBufferPositionedRead() throws IOException { - try (FSDataInputStream inputStream = fs.open(filePath)) { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testByteBufferPositionedRead(boolean isStreamEnable) throws IOException { + OzoneConfiguration conf = cluster().getConf(); + conf.setBoolean("ozone.client.stream.readblock.enable", isStreamEnable); + try (FileSystem fs = FileSystem.get(URI.create(uri), conf); + FSDataInputStream inputStream = fs.open(filePath)) { int bufferCapacity = 20; ByteBuffer buffer = ByteBuffer.allocate(bufferCapacity); long currentPos = inputStream.getPos(); @@ -182,9 +195,12 @@ public void testByteBufferPositionedRead() throws IOException { } @ParameterizedTest - @ValueSource(ints = { -1, 30 * 1024 * 1024, 30 * 1024 * 1024 + 1 }) - public void testByteBufferPositionedReadWithInvalidPosition(int position) throws IOException { - try (FSDataInputStream inputStream = fs.open(filePath)) { + @MethodSource("isStreamEnableAndData") + public void testByteBufferPositionedReadWithInvalidPosition(boolean isStreamEnable, int position) throws IOException { + OzoneConfiguration conf = cluster().getConf(); + conf.setBoolean("ozone.client.stream.readblock.enable", isStreamEnable); + try (FileSystem fs = FileSystem.get(URI.create(uri), conf); + FSDataInputStream inputStream = fs.open(filePath)) { long currentPos = inputStream.getPos(); ByteBuffer buffer = ByteBuffer.allocate(20); assertEquals(-1, inputStream.read(position, buffer)); @@ -193,9 +209,13 @@ public void testByteBufferPositionedReadWithInvalidPosition(int position) throws } } - @Test - public void testByteBufferPositionedReadFully() throws IOException { - try (FSDataInputStream inputStream = fs.open(filePath)) { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testByteBufferPositionedReadFully(boolean isStreamEnable) throws IOException { + OzoneConfiguration conf = cluster().getConf(); + conf.setBoolean("ozone.client.stream.readblock.enable", isStreamEnable); + try (FileSystem fs = FileSystem.get(URI.create(uri), conf); + FSDataInputStream inputStream = fs.open(filePath)) { int bufferCapacity = 20; long currentPos = inputStream.getPos(); ByteBuffer buffer = ByteBuffer.allocate(bufferCapacity); @@ -235,9 +255,13 @@ public void testByteBufferPositionedReadFully() throws IOException { } @ParameterizedTest - @ValueSource(ints = { -1, 30 * 1024 * 1024, 30 * 1024 * 1024 + 1 }) - public void testByteBufferPositionedReadFullyWithInvalidPosition(int position) throws IOException { - try (FSDataInputStream inputStream = fs.open(filePath)) { + @MethodSource("isStreamEnableAndData") + public void testByteBufferPositionedReadFullyWithInvalidPosition( + boolean isStreamEnable, int position) throws IOException { + OzoneConfiguration conf = cluster().getConf(); + conf.setBoolean("ozone.client.stream.readblock.enable", isStreamEnable); + try (FileSystem fs = FileSystem.get(URI.create(uri), conf); + FSDataInputStream inputStream = fs.open(filePath)) { long currentPos = inputStream.getPos(); ByteBuffer buffer = ByteBuffer.allocate(20); assertThrows(EOFException.class, () -> inputStream.readFully(position, buffer)); @@ -246,9 +270,13 @@ public void testByteBufferPositionedReadFullyWithInvalidPosition(int position) t } } - @Test - public void testO3FSMultiByteRead() throws IOException { - try (FSDataInputStream inputStream = fs.open(filePath)) { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testO3FSMultiByteRead(boolean isStreamEnable) throws IOException { + OzoneConfiguration conf = cluster().getConf(); + conf.setBoolean("ozone.client.stream.readblock.enable", isStreamEnable); + try (FileSystem fs = FileSystem.get(URI.create(uri), conf); + FSDataInputStream inputStream = fs.open(filePath)) { byte[] value = new byte[data.length]; byte[] tmp = new byte[1 * 1024 * 1024]; int i = 0; @@ -265,10 +293,13 @@ public void testO3FSMultiByteRead() throws IOException { } } - @Test - public void testO3FSByteBufferRead() throws IOException { - try (FSDataInputStream inputStream = fs.open(filePath)) { - + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testO3FSByteBufferRead(boolean isStreamEnable) throws IOException { + OzoneConfiguration conf = cluster().getConf(); + conf.setBoolean("ozone.client.stream.readblock.enable", isStreamEnable); + try (FileSystem fs = FileSystem.get(URI.create(uri), conf); + FSDataInputStream inputStream = fs.open(filePath)) { ByteBuffer buffer = ByteBuffer.allocate(1024 * 1024); int byteRead = inputStream.read(buffer); @@ -281,29 +312,34 @@ public void testO3FSByteBufferRead() throws IOException { } } - @Test - public void testSequenceFileReaderSync() throws IOException { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testSequenceFileReaderSync(boolean isStreamEnable) throws IOException { File srcfile = new File("src/test/resources/testSequenceFile"); Path path = new Path("/" + RandomStringUtils.secure().nextAlphanumeric(5)); InputStream input = new BufferedInputStream(Files.newInputStream(srcfile.toPath())); // Upload test SequenceFile file - FSDataOutputStream output = fs.create(path); - IOUtils.copyBytes(input, output, 4096, true); - input.close(); + OzoneConfiguration conf = cluster().getConf(); + conf.setBoolean("ozone.client.stream.readblock.enable", isStreamEnable); + try (FileSystem fs = FileSystem.get(URI.create(uri), conf); + FSDataOutputStream output = fs.create(path);) { + IOUtils.copyBytes(input, output, 4096, true); + input.close(); - // Start SequenceFile.Reader test - SequenceFile.Reader in = new SequenceFile.Reader(fs, path, cluster().getConf()); - long blockStart = -1; - // EOFException should not occur. - in.sync(0); - blockStart = in.getPosition(); - // The behavior should be consistent with HDFS - assertEquals(srcfile.length(), blockStart); - in.close(); + // Start SequenceFile.Reader test + SequenceFile.Reader in = new SequenceFile.Reader(fs, path, cluster().getConf()); + long blockStart = -1; + // EOFException should not occur. + in.sync(0); + blockStart = in.getPosition(); + // The behavior should be consistent with HDFS + assertEquals(srcfile.length(), blockStart); + } } - @Test + @ParameterizedTest + @ValueSource(booleans = {true, false}) public void testSequenceFileReaderSyncEC() throws IOException { File srcfile = new File("src/test/resources/testSequenceFile"); Path path = new Path("/" + RandomStringUtils.secure().nextAlphanumeric(5)); @@ -324,4 +360,15 @@ public void testSequenceFileReaderSyncEC() throws IOException { assertEquals(srcfile.length(), blockStart); in.close(); } + + static Stream<Arguments> isStreamEnableAndData() { + return Stream.of( + Arguments.of(false, -1), + Arguments.of(false, 30 * 1024 * 1024), + Arguments.of(false, 30 * 1024 * 1024 + 1), + Arguments.of(true, -1), + Arguments.of(true, 30 * 1024 * 1024), + Arguments.of(true, 30 * 1024 * 1024 + 1) + ); + } }
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java index 012c7a6..1bd4b1b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java
@@ -49,8 +49,8 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.OmUtils; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneKey; @@ -104,7 +104,7 @@ public void init() throws Exception { bucketName = RandomStringUtils.secure().nextAlphabetic(10).toLowerCase(); // create a volume and a bucket to be used by OzoneFileSystem - TestDataUtil.createVolumeAndBucket(client, volumeName, bucketName, BucketLayout.LEGACY); + DataTestUtil.createVolumeAndBucket(client, volumeName, bucketName, BucketLayout.LEGACY); String rootPath = String.format("%s://%s.%s/", OZONE_URI_SCHEME, bucketName, volumeName); @@ -402,7 +402,7 @@ public void testDoubleSlashPrefixPathNormalization(int slashCount) throws Except ArrayList<String> expectedKeys = new ArrayList<>(); expectedKeys.add(dirPath); expectedKeys.add(normalizedKey); - TestDataUtil.createKey(ozoneBucket, slashyKey, data); + DataTestUtil.createKey(ozoneBucket, slashyKey, data); try { ozoneBucket.readKey(slashyKey).close(); @@ -430,7 +430,7 @@ private void checkKeyList(Iterator<? extends OzoneKey > ozoneKeyIterator, private void createAndAssertKey(OzoneBucket ozoneBucket, String key, int length) throws Exception { - byte[] input = TestDataUtil.createStringKey(ozoneBucket, key, length); + byte[] input = DataTestUtil.createStringKey(ozoneBucket, key, length); // Read the key with given key name. readKey(ozoneBucket, key, length, input);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileChecksum.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileChecksum.java index 20bc7bb..45a8553 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileChecksum.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileChecksum.java
@@ -19,9 +19,9 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY; +import static org.apache.hadoop.ozone.DataTestUtil.createBucket; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_NETWORK_TOPOLOGY_AWARE_READ_KEY; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE; -import static org.apache.hadoop.ozone.TestDataUtil.createBucket; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.params.provider.Arguments.arguments;
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemMetrics.java index b57e298..8a4e824 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemMetrics.java
@@ -28,8 +28,8 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; @@ -70,7 +70,7 @@ void init() throws Exception { omConfig.setFileSystemPathEnabled(true); // create a volume and a bucket to be used by OzoneFileSystem - bucket = TestDataUtil.createVolumeAndBucket(client, BucketLayout.LEGACY); + bucket = DataTestUtil.createVolumeAndBucket(client, BucketLayout.LEGACY); // Set the fs.defaultFS and start the filesystem String uri = String.format("%s://%s.%s/",
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemMissingParent.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemMissingParent.java index fc79197..1bea1f1 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemMissingParent.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemMissingParent.java
@@ -27,8 +27,8 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -54,7 +54,7 @@ public abstract class TestOzoneFileSystemMissingParent implements NonHATests.Tes void init() throws Exception { client = cluster().newClient(); - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client); + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client); String volumeName = bucket.getVolumeName(); Path volumePath = new Path(OZONE_URI_DELIMITER, volumeName);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemPrefixParser.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemPrefixParser.java index facea44..7401e4b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemPrefixParser.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemPrefixParser.java
@@ -27,9 +27,9 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.debug.om.PrefixParser; import org.apache.hadoop.ozone.om.OMStorage; @@ -70,7 +70,7 @@ public static void init() throws Exception { // create a volume and a bucket to be used by OzoneFileSystem try (OzoneClient client = cluster.newClient()) { - TestDataUtil.createVolumeAndBucket(client, volumeName, bucketName, + DataTestUtil.createVolumeAndBucket(client, volumeName, bucketName, BucketLayout.FILE_SYSTEM_OPTIMIZED); }
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreaming.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreaming.java index c141f3f..f85247d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreaming.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreaming.java
@@ -44,8 +44,8 @@ import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.ClientConfigForTesting; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.io.SelectorOutputStream; @@ -101,7 +101,7 @@ public static void init() throws Exception { client = cluster.newClient(); // create a volume and a bucket to be used by OzoneFileSystem - bucket = TestDataUtil.createVolumeAndBucket(client, layout); + bucket = DataTestUtil.createVolumeAndBucket(client, layout); } @AfterAll
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreamingDisabledDatanode.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreamingDisabledDatanode.java index cb59a3a..8019e3f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreamingDisabledDatanode.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreamingDisabledDatanode.java
@@ -35,8 +35,8 @@ import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.ClientConfigForTesting; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -85,7 +85,7 @@ public static void init() throws Exception { cluster.waitForClusterToBeReady(); client = cluster.newClient(); - bucket = TestDataUtil.createVolumeAndBucket(client, layout); + bucket = DataTestUtil.createVolumeAndBucket(client, layout); } @AfterAll
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java index 87ad01b..b15bf14 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
@@ -21,9 +21,9 @@ import java.io.IOException; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -51,7 +51,7 @@ public Path getTestPath() { protected String getRootURI() throws IOException { try (OzoneClient client = getCluster().newClient()) { BucketLayout layout = getConf().getEnum(OZONE_DEFAULT_BUCKET_LAYOUT, BucketLayout.DEFAULT); - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client, layout); + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client, layout); return String.format("%s://%s.%s/", getScheme(), bucket.getName(), bucket.getVolumeName()); } }
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestCloseContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestCloseContainer.java index 7cbe2cc..a3383a7 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestCloseContainer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestCloseContainer.java
@@ -50,10 +50,10 @@ import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager.ReplicationManagerConfiguration; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneTestUtils; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeTestUtils; @@ -106,7 +106,7 @@ public void setUp() throws Exception { cluster.waitForClusterToBeReady(); client = cluster.newClient(); - bucket = TestDataUtil.createVolumeAndBucket(client, volName, bucketName); + bucket = DataTestUtil.createVolumeAndBucket(client, volName, bucketName); } @AfterEach @@ -122,7 +122,7 @@ public void testReplicasAreReportedForClosedContainerAfterRestart() throws Exception { // Create some keys to write data into the open containers for (int i = 0; i < 10; i++) { - TestDataUtil.createKey(bucket, "key" + i, "this is the content".getBytes(UTF_8)); + DataTestUtil.createKey(bucket, "key" + i, "this is the content".getBytes(UTF_8)); } StorageContainerManager scm = cluster.getStorageContainerManager(); @@ -180,7 +180,7 @@ public void testCloseClosedContainer() throws Exception { // Create some keys to write data into the open containers for (int i = 0; i < 10; i++) { - TestDataUtil.createKey(bucket, "key" + i, "this is the content".getBytes(UTF_8)); + DataTestUtil.createKey(bucket, "key" + i, "this is the content".getBytes(UTF_8)); } StorageContainerManager scm = cluster.getStorageContainerManager(); // Pick any container on the cluster and close it via client @@ -213,7 +213,7 @@ public void testCloseClosedContainer() public void testContainerChecksumForClosedContainer() throws Exception { // Create some keys to write data into the open containers ReplicationConfig repConfig = RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE); - TestDataUtil.createKey(bucket, "key1", repConfig, "this is the content".getBytes(UTF_8)); + DataTestUtil.createKey(bucket, "key1", repConfig, "this is the content".getBytes(UTF_8)); StorageContainerManager scm = cluster.getStorageContainerManager(); ContainerInfo containerInfo1 = scm.getContainerManager().getContainers().get(0); @@ -243,7 +243,7 @@ public void testContainerChecksumForClosedContainer() throws Exception { } // Create 2nd container and check the checksum doesn't match with 1st container - TestDataUtil.createKey(bucket, "key2", repConfig, "this is the different content".getBytes(UTF_8)); + DataTestUtil.createKey(bucket, "key2", repConfig, "this is the different content".getBytes(UTF_8)); ContainerInfo containerInfo2 = scm.getContainerManager().getContainers().get(1); for (HddsDatanodeService hddsDatanode : hddsDatanodes) { assertFalse(containerChecksumFileExists(hddsDatanode, containerInfo2.getContainerID()));
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestContainerOperations.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestContainerOperations.java index 8f46d73..6740a1e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestContainerOperations.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestContainerOperations.java
@@ -97,7 +97,7 @@ void testContainerStateMachineIdempotency() throws Exception { // call create Container again BlockID blockID = ContainerTestHelper.getTestBlockID(containerID); byte[] data = - RandomStringUtils.secure().next(RandomUtils.secure().randomInt(0, 1024)).getBytes(UTF_8); + RandomStringUtils.secure().next(RandomUtils.secure().randomInt(1, 1024)).getBytes(UTF_8); ContainerProtos.ContainerCommandRequestProto writeChunkRequest = ContainerTestHelper .getWriteChunkRequest(container.getPipeline(), blockID,
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestFailoverWithSCMHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestFailoverWithSCMHA.java index bbbee76..c2c7b3c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestFailoverWithSCMHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestFailoverWithSCMHA.java
@@ -171,6 +171,7 @@ public void testContainerBalancerPersistsConfigurationInAllSCMs() ScmClient scmClient = new ContainerOperationClient(conf); // assert that container balancer is not running right now assertFalse(scmClient.getContainerBalancerStatus()); + conf.setInt("hdds.container.balancer.datanodes.involved.max.percentage.per.iteration", 100); ContainerBalancerConfiguration balancerConf = conf.getObject(ContainerBalancerConfiguration.class); ContainerBalancer containerBalancer = leader.getContainerBalancer();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestRackAwarePlacement.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestRackAwarePlacement.java new file mode 100644 index 0000000..c8258eb --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestRackAwarePlacement.java
@@ -0,0 +1,412 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; +import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.container.ContainerInfo; +import org.apache.hadoop.hdds.scm.container.ContainerReplica; +import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; +import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.net.NetworkTopology; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.client.ObjectStore; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.apache.ozone.test.GenericTestUtils; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Integration tests that verify rack/host topology is correctly propagated + * to SCM and that pipeline and container placement respect rack boundaries. + * + */ +public class TestRackAwarePlacement { + + private static final String RACK0 = "/rack0"; + private static final String RACK1 = "/rack1"; + + private static final String[] RACKS = { + RACK0, RACK0, RACK0, + RACK1, RACK1, RACK1 + }; + + private static final String[] HOSTS = { + "host0.test", "host1.test", "host2.test", + "host3.test", "host4.test", "host5.test" + }; + + private static void applyReplicationSpeedupConfig(OzoneConfiguration conf) { + conf.setTimeDuration(ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, + 100, TimeUnit.MILLISECONDS); + conf.setTimeDuration(ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL, + 3, TimeUnit.SECONDS); + conf.setTimeDuration(ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL, + 6, TimeUnit.SECONDS); + conf.setTimeDuration("hdds.scm.replication.thread.interval", + 1, TimeUnit.SECONDS); + conf.setTimeDuration("hdds.scm.replication.under.replicated.interval", + 5, TimeUnit.SECONDS); + conf.setTimeDuration("hdds.scm.replication.over.replicated.interval", + 5, TimeUnit.SECONDS); + } + + static Stream<Arguments> rackAwarePolicies() { + return Stream.of( + Arguments.of( + "org.apache.hadoop.hdds.scm.container.placement.algorithms" + + ".SCMContainerPlacementRackAware"), + Arguments.of( + "org.apache.hadoop.hdds.scm.container.placement.algorithms" + + ".SCMContainerPlacementRackScatter") + ); + } + + @ParameterizedTest + @MethodSource("rackAwarePolicies") + void testContainerPlacementWithPolicy( + String placementClassName) throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY, + placementClassName); + applyReplicationSpeedupConfig(conf); + + try (MiniOzoneCluster cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(RACKS.length) + .setRacks(RACKS) + .setHosts(HOSTS) + .build()) { + cluster.waitForClusterToBeReady(); + cluster.waitForPipelineTobeReady(ReplicationFactor.THREE, 60_000); + + StorageContainerManager scm = cluster.getStorageContainerManager(); + PlacementPolicy actualPolicy = scm.getContainerPlacementPolicy(); + assertEquals(placementClassName, actualPolicy.getClass().getName(), + "Placement policy was not set correctly"); + + assertPipelinesSpanMultipleRacks(cluster); + assertContainerReplicationIsRackAware(cluster); + } + } + + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + class WithRacksAndHosts { + + private MiniOzoneCluster cluster; + + @BeforeAll + void init() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + applyReplicationSpeedupConfig(conf); + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(RACKS.length) + .setRacks(RACKS) + .setHosts(HOSTS) + .build(); + cluster.waitForClusterToBeReady(); + cluster.waitForPipelineTobeReady(ReplicationFactor.THREE, 60_000); + } + + @AfterAll + void tearDown() { + if (cluster != null) { + cluster.shutdown(); + } + } + + @Test + void testDatanodesHaveCorrectRack() { + assertRackAssignments(cluster, RACKS); + } + + @Test + void testDatanodesHaveCorrectHostname() { + assertHostnameAssignments(cluster, HOSTS); + } + + @Test + void testRatisPipelineSpansMultipleRacks() { + assertPipelinesSpanMultipleRacks(cluster); + } + + @Test + void testContainerReplicationIsRackAware() throws Exception { + assertContainerReplicationIsRackAware(cluster); + } + } + + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + class WithRacksOnly { + + private MiniOzoneCluster cluster; + + @BeforeAll + void init() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + applyReplicationSpeedupConfig(conf); + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(RACKS.length) + .setRacks(RACKS) + .build(); + cluster.waitForClusterToBeReady(); + cluster.waitForPipelineTobeReady(ReplicationFactor.THREE, 60_000); + } + + @AfterAll + void tearDown() { + if (cluster != null) { + cluster.shutdown(); + } + } + + @Test + void testDatanodesHaveCorrectRack() { + assertRackAssignments(cluster, RACKS); + } + + @Test + void testRatisPipelineSpansMultipleRacks() { + assertPipelinesSpanMultipleRacks(cluster); + } + + @Test + void testContainerReplicationIsRackAware() throws Exception { + assertContainerReplicationIsRackAware(cluster); + } + } + + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + class WithHostsOnly { + + private MiniOzoneCluster cluster; + + @BeforeAll + void init() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(HOSTS.length) + .setHosts(HOSTS) + .build(); + cluster.waitForClusterToBeReady(); + cluster.waitForPipelineTobeReady(ReplicationFactor.THREE, 60_000); + } + + @AfterAll + void tearDown() { + if (cluster != null) { + cluster.shutdown(); + } + } + + @Test + void testDatanodesHaveCorrectHostname() { + assertHostnameAssignments(cluster, HOSTS); + } + + @Test + void testDatanodesAllInDefaultRack() { + NodeManager nodeManager = + cluster.getStorageContainerManager().getScmNodeManager(); + List<? extends DatanodeDetails> allNodes = nodeManager.getAllNodes(); + + for (DatanodeDetails dn : allNodes) { + assertEquals(NetworkTopology.DEFAULT_RACK, dn.getNetworkLocation(), + "Datanode " + dn.getHostName() + + " should be in default rack when no racks are configured"); + } + } + } + + private static void assertContainerReplicationIsRackAware( + MiniOzoneCluster cluster) throws Exception { + StorageContainerManager scm = cluster.getStorageContainerManager(); + + try (OzoneClient client = cluster.newClient()) { + ObjectStore store = client.getObjectStore(); + store.createVolume("testvol"); + OzoneVolume volume = store.getVolume("testvol"); + volume.createBucket("testbucket"); + OzoneBucket bucket = volume.getBucket("testbucket"); + + byte[] data = "test-data".getBytes(StandardCharsets.UTF_8); + try (OzoneOutputStream out = bucket.createKey( + "testkey", data.length, + RatisReplicationConfig.getInstance(ReplicationFactor.THREE), + new HashMap<>())) { + out.write(data); + } + } + + ContainerInfo targetContainer = null; + Set<ContainerReplica> replicas = null; + for (ContainerInfo c : scm.getContainerManager().getContainers()) { + Set<ContainerReplica> r = + scm.getContainerManager().getContainerReplicas(c.containerID()); + if (r.size() >= 3) { + targetContainer = c; + replicas = r; + break; + } + } + assertNotNull(targetContainer, + "Should find a container with 3 replicas"); + ContainerID containerID = targetContainer.containerID(); + + DatanodeDetails stoppedDn = + replicas.iterator().next().getDatanodeDetails(); + cluster.shutdownHddsDatanode(stoppedDn); + + GenericTestUtils.waitFor(() -> { + try { + return scm.getScmNodeManager() + .getNodeStatus(stoppedDn) + .getHealth() == HddsProtos.NodeState.DEAD; + } catch (Exception e) { + return false; + } + }, 500, 30_000); + + GenericTestUtils.waitFor(() -> { + try { + return scm.getContainerManager() + .getContainerReplicas(containerID) + .size() >= 3; + } catch (Exception e) { + return false; + } + }, 1_000, 60_000); + + Set<String> racks = scm.getContainerManager() + .getContainerReplicas(containerID) + .stream() + .map(r -> r.getDatanodeDetails().getNetworkLocation()) + .collect(Collectors.toSet()); + + assertTrue(racks.size() >= 2, + "Container replicas after re-replication should span at least " + + "2 racks, but were on: " + racks); + } + + private static void assertRackAssignments(MiniOzoneCluster cluster, + String[] expectedRacks) { + NodeManager nodeManager = + cluster.getStorageContainerManager().getScmNodeManager(); + List<? extends DatanodeDetails> allNodes = nodeManager.getAllNodes(); + + assertEquals(expectedRacks.length, allNodes.size(), + "Number of registered datanodes should match number of configured racks"); + + long actualRack0 = allNodes.stream() + .filter(dn -> RACK0.equals(dn.getNetworkLocation())) + .count(); + long actualRack1 = allNodes.stream() + .filter(dn -> RACK1.equals(dn.getNetworkLocation())) + .count(); + + long expectedRack0 = + Arrays.stream(expectedRacks).filter(RACK0::equals).count(); + long expectedRack1 = + Arrays.stream(expectedRacks).filter(RACK1::equals).count(); + + assertEquals(expectedRack0, actualRack0, + "Expected " + expectedRack0 + " datanodes on " + RACK0); + assertEquals(expectedRack1, actualRack1, + "Expected " + expectedRack1 + " datanodes on " + RACK1); + + for (DatanodeDetails dn : allNodes) { + String location = dn.getNetworkLocation(); + assertNotNull(location, + "Network location must not be null for " + dn.getHostName()); + assertTrue(location.equals(RACK0) || location.equals(RACK1), + "Unexpected rack for datanode " + dn.getHostName() + + ": " + location); + } + } + + private static void assertHostnameAssignments(MiniOzoneCluster cluster, + String[] expectedHosts) { + NodeManager nodeManager = + cluster.getStorageContainerManager().getScmNodeManager(); + List<? extends DatanodeDetails> allNodes = nodeManager.getAllNodes(); + + assertEquals(expectedHosts.length, allNodes.size(), + "Number of registered datanodes should match number of configured hosts"); + + Set<String> actual = allNodes.stream() + .map(DatanodeDetails::getHostName) + .collect(Collectors.toSet()); + + Set<String> expected = Arrays.stream(expectedHosts) + .collect(Collectors.toSet()); + + assertEquals(expected, actual, + "Registered datanode hostnames should match configured hosts"); + } + + private static void assertPipelinesSpanMultipleRacks( + MiniOzoneCluster cluster) { + List<Pipeline> pipelines = cluster.getStorageContainerManager() + .getPipelineManager() + .getPipelines( + RatisReplicationConfig.getInstance(ReplicationFactor.THREE), + Pipeline.PipelineState.OPEN); + + assertFalse(pipelines.isEmpty(), + "There should be at least one open RATIS THREE pipeline"); + + for (Pipeline pipeline : pipelines) { + Set<String> racks = pipeline.getNodes().stream() + .map(DatanodeDetails::getNetworkLocation) + .collect(Collectors.toSet()); + + assertTrue(racks.size() >= 2, + "Pipeline " + pipeline.getId() + + " should span at least 2 racks, but spans: " + racks); + } + } +}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMFollowerCatchupWithContainerReport.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMFollowerCatchupWithContainerReport.java index 1e172d7..8ab1f6a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMFollowerCatchupWithContainerReport.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMFollowerCatchupWithContainerReport.java
@@ -34,9 +34,9 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -280,7 +280,7 @@ private Set<Long> createKeys(String volumeName, String bucketName, byte[] keyDat for (int i = 0; i < NUM_KEYS; i++) { String keyName = "key-" + i; - TestDataUtil.createKey(bucket, keyName, + DataTestUtil.createKey(bucket, keyName, RatisReplicationConfig.getInstance(THREE), keyData); OzoneKeyDetails keyDetails = bucket.getKey(keyName); keyDetails.getOzoneKeyLocations()
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManager.java index 1651571..b21857b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManager.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManager.java
@@ -115,12 +115,12 @@ import org.apache.hadoop.net.DNSToSwitchMapping; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.net.StaticMapping; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneTestUtils; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.common.DeletedBlock; import org.apache.hadoop.ozone.container.ContainerTestHelper; import org.apache.hadoop.ozone.container.common.helpers.BlockData; @@ -265,7 +265,7 @@ private void testBlockDeletionTransactions(MiniOzoneCluster cluster) throws Exce .getScmBlockManager().getDeletedBlockLog(); assertEquals(0, delLog.getNumOfValidTransactions()); - Map<String, OmKeyInfo> keyLocations = TestDataUtil.createKeys(cluster, KEY_COUNT); + Map<String, OmKeyInfo> keyLocations = DataTestUtil.createKeys(cluster, KEY_COUNT); // Wait for container report Thread.sleep(1000); for (OmKeyInfo keyInfo : keyLocations.values()) { @@ -438,7 +438,7 @@ public void testBlockDeletingThrottling() throws Exception { .getScmBlockManager().getSCMBlockDeletingService(); delService.setBlockDeleteTXNum(limitSize); - Map<String, OmKeyInfo> keyLocations = TestDataUtil.createKeys(cluster, numKeys); + Map<String, OmKeyInfo> keyLocations = DataTestUtil.createKeys(cluster, numKeys); // Wait for container report Thread.sleep(5000); for (OmKeyInfo keyInfo : keyLocations.values()) { @@ -674,7 +674,7 @@ public void testCloseContainerCommandOnRestart() throws Exception { cluster.waitForClusterToBeReady(); cluster.waitForPipelineTobeReady(HddsProtos.ReplicationFactor.ONE, 30000); - TestDataUtil.createKeys(cluster, 10); + DataTestUtil.createKeys(cluster, 10); GenericTestUtils.waitFor(() -> cluster.getStorageContainerManager().getContainerManager() .getContainers() != null, 1000, 10000);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHAWithAllRunning.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHAWithAllRunning.java index 9a3722c..c3bba52 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHAWithAllRunning.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHAWithAllRunning.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.hdds.scm.ha.SCMHAMetrics; import org.apache.hadoop.hdds.scm.ha.SCMRatisServerImpl; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; -import org.apache.hadoop.ozone.TestDataUtil; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -126,7 +126,7 @@ private void doPutKey() throws Exception { byte[] bytes = value.getBytes(UTF_8); RatisReplicationConfig replication = RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE); - TestDataUtil.createKey(bucket, keyName, replication, bytes); + DataTestUtil.createKey(bucket, keyName, replication, bytes); OzoneKey key = bucket.getKey(keyName); assertEquals(keyName, key.getName());
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestPendingContainerTrackerIntegration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestPendingContainerTrackerIntegration.java index 13d6bf4..657e5d0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestPendingContainerTrackerIntegration.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestPendingContainerTrackerIntegration.java
@@ -30,8 +30,8 @@ import org.apache.hadoop.hdds.scm.node.SCMNodeManager; import org.apache.hadoop.hdds.scm.node.SCMNodeMetrics; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; @@ -82,7 +82,7 @@ public void setup() throws Exception { client = cluster.newClient(); // Create bucket for testing - bucket = TestDataUtil.createVolumeAndBucket(client); + bucket = DataTestUtil.createVolumeAndBucket(client); SCMNodeManager nodeManager = (SCMNodeManager) scm.getScmNodeManager(); assertNotNull(nodeManager);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/metrics/TestSCMContainerManagerMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/metrics/TestSCMContainerManagerMetrics.java index 71d8e39..1391e58 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/metrics/TestSCMContainerManagerMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/metrics/TestSCMContainerManagerMetrics.java
@@ -35,9 +35,9 @@ import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneTestUtils; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.NonHATests; @@ -137,7 +137,7 @@ public void testReportProcessingMetrics() throws Exception { OzoneTestUtils.closeAllContainers(scm.getEventQueue(), scm); // Create key should create container on DN. - TestDataUtil.createKeys(cluster(), 1); + DataTestUtil.createKeys(cluster(), 1); GenericTestUtils.waitFor(() -> { final MetricsRecordBuilder scmMetrics =
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManagerIntegration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManagerIntegration.java index a1855fe..65ce765 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManagerIntegration.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManagerIntegration.java
@@ -70,9 +70,9 @@ import org.apache.hadoop.hdds.scm.node.NodeManager; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneTestUtils; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneKeyDetails; @@ -157,7 +157,7 @@ void init() throws Exception { client = cluster.newClient(); scmClient = new ContainerOperationClient(cluster.getConf()); - bucket = TestDataUtil.createVolumeAndBucket(client); + bucket = DataTestUtil.createVolumeAndBucket(client); } @AfterEach @@ -178,7 +178,7 @@ void testReplicationManagerNotify() throws Exception { public void testClosedContainerReplicationWhenNodeDies() throws Exception { String keyName = "key-" + UUID.randomUUID(); - TestDataUtil.createKey(bucket, keyName, RATIS_REPLICATION_CONFIG, + DataTestUtil.createKey(bucket, keyName, RATIS_REPLICATION_CONFIG, "this is the content".getBytes(StandardCharsets.UTF_8)); // Get the container ID for the key @@ -226,7 +226,7 @@ void testClosedContainerReplicationWhenNodeDecommissionAndBackToInService( throws Exception { String keyName = "key-" + UUID.randomUUID(); - TestDataUtil.createKey(bucket, keyName, RATIS_REPLICATION_CONFIG, + DataTestUtil.createKey(bucket, keyName, RATIS_REPLICATION_CONFIG, "this is the content".getBytes(StandardCharsets.UTF_8)); OzoneKeyDetails key = bucket.getKey(keyName); @@ -277,7 +277,7 @@ void testClosedContainerReplicationWhenNodeDecommissionAndBackToInService( @Test public void testDeadMaintenanceNodeAndDecommission() throws Exception { String keyName = "key-" + UUID.randomUUID(); - TestDataUtil.createKey(bucket, keyName, RATIS_REPLICATION_CONFIG, + DataTestUtil.createKey(bucket, keyName, RATIS_REPLICATION_CONFIG, "this is the content".getBytes(StandardCharsets.UTF_8)); OzoneKeyDetails key = bucket.getKey(keyName); @@ -321,7 +321,7 @@ public void testDeadMaintenanceNodeAndDecommission() throws Exception { @Test public void testOneDeadMaintenanceNodeAndOneLiveMaintenanceNodeAndOneDecommissionNode() throws Exception { String keyName = "key-" + UUID.randomUUID(); - TestDataUtil.createKey(bucket, keyName, RATIS_REPLICATION_CONFIG, + DataTestUtil.createKey(bucket, keyName, RATIS_REPLICATION_CONFIG, "this is the content".getBytes(StandardCharsets.UTF_8)); OzoneKeyDetails key = bucket.getKey(keyName);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/node/TestDecommissionAndMaintenance.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/node/TestDecommissionAndMaintenance.java index 7ccc131..5260c97 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/node/TestDecommissionAndMaintenance.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/node/TestDecommissionAndMaintenance.java
@@ -73,10 +73,10 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneClusterProvider; import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.ozone.test.GenericTestUtils; @@ -165,7 +165,7 @@ public void setUp() throws Exception { cluster = clusterProvider.provide(); setManagers(); client = cluster.newClient(); - bucket = TestDataUtil.createVolumeAndBucket(client, volName, bucketName); + bucket = DataTestUtil.createVolumeAndBucket(client, volName, bucketName); scmClient = new ContainerOperationClient(cluster.getConf()); } @@ -831,7 +831,7 @@ private void setManagers() { private void generateData(int keyCount, String keyPrefix, ReplicationConfig replicationConfig) throws IOException { for (int i = 0; i < keyCount; i++) { - TestDataUtil.createKey(bucket, keyPrefix + i, replicationConfig, + DataTestUtil.createKey(bucket, keyPrefix + i, replicationConfig, "this is the content".getBytes(StandardCharsets.UTF_8)); } }
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java index 4a74d94..ef3c1e2 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java
@@ -45,9 +45,9 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.ozone.test.GenericTestUtils; @@ -186,11 +186,11 @@ void testSafeModeExitAfterScmRestartWithMixedEcAndRatisThreeKeys() waitForRatis3NodePipelines(1); try (OzoneClient client = cluster.newClient()) { - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client); - TestDataUtil.createKey(bucket, "ec-key", + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client); + DataTestUtil.createKey(bucket, "ec-key", new ECReplicationConfig("rs-3-2-1024k"), "ec-data".getBytes(UTF_8)); - TestDataUtil.createKey(bucket, "ratis3-key", + DataTestUtil.createKey(bucket, "ratis3-key", RatisReplicationConfig.getInstance(ReplicationFactor.THREE), "ratis-data".getBytes(UTF_8)); }
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeSCMHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeSCMHA.java index 6f85c09..84da96e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeSCMHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeSCMHA.java
@@ -26,9 +26,9 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.ha.SCMStateMachine; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -118,7 +118,7 @@ private void createTestData(OzoneClient client) throws IOException { OzoneBucket bucket = volume.getBucket("testbucket"); - TestDataUtil.createKey(bucket, "testkey123", + DataTestUtil.createKey(bucket, "testkey123", RatisReplicationConfig.getInstance(THREE), "Hello".getBytes(UTF_8)); } }
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java index d082019..a1a8ca8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java
@@ -68,9 +68,9 @@ import org.apache.hadoop.hdds.utils.db.CodecException; import org.apache.hadoop.hdds.utils.db.RocksDatabaseException; import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.UniformDatanodesFactory; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.ObjectStore; @@ -335,7 +335,7 @@ public void testFinalizationNonEmptyClusterDataDistribution() throws Exception { final String keyName = "key" + System.nanoTime(); // Create the key String value = "sample value"; - TestDataUtil.createKey(bucket, keyName, ReplicationConfig.fromTypeAndFactor(RATIS, THREE), value.getBytes(UTF_8)); + DataTestUtil.createKey(bucket, keyName, ReplicationConfig.fromTypeAndFactor(RATIS, THREE), value.getBytes(UTF_8)); // update scmInfo in OM OzoneKeyDetails keyDetails = bucket.getKey(keyName); // delete the key
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/DataTestUtil.java similarity index 98% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java rename to hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/DataTestUtil.java index dc0735c..b1d10d9 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/DataTestUtil.java
@@ -54,11 +54,11 @@ import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; /** - * Utility to help to generate test data. + * Utility class with helper methods for creating and managing test data in integration tests. */ -public final class TestDataUtil { +public final class DataTestUtil { - private TestDataUtil() { + private DataTestUtil() { } public static OzoneBucket createVolumeAndBucket(OzoneClient client,
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java index bb8d070..80d2c11 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java
@@ -25,17 +25,21 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.PlacementPolicy; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient; import org.apache.hadoop.hdds.scm.client.ScmClient; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancerConfiguration; +import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity; import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.AfterAll; @@ -84,7 +88,7 @@ public void testContainerBalancerCLIOperations() throws Exception { Optional<Integer> iterations = Optional.of(10000); Optional<Integer> maxDatanodesPercentageToInvolvePerIteration = Optional.of(100); - Optional<Long> maxSizeToMovePerIterationInGB = Optional.of(1L); + Optional<Long> maxSizeToMovePerIterationInGB = Optional.of(6L); Optional<Long> maxSizeEnteringTargetInGB = Optional.of(6L); Optional<Long> maxSizeLeavingSourceInGB = Optional.of(6L); Optional<Integer> balancingInterval = Optional.of(70); @@ -149,14 +153,24 @@ public void testIfCBCLIOverridesConfigs() throws Exception { //CLI option for iterations and balancing interval is not passed Optional<Integer> iterations = Optional.empty(); Optional<Integer> balancingInterval = Optional.empty(); - String excludedContainersList = "1,2,3"; - String includedContainersList = "4,5"; + List<ContainerWithPipeline> createdContainers = new ArrayList<>(5); + for (int i = 0; i < 5; i++) { + createdContainers.add(containerBalancerClient.createContainer( + HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.ONE, + OzoneConsts.OZONE)); + } + String excludedContainersList = createdContainers.get(0).getContainerInfo().getContainerID() + "," + + createdContainers.get(1).getContainerInfo().getContainerID() + "," + + createdContainers.get(2).getContainerInfo().getContainerID(); + String includedContainersList = createdContainers.get(3).getContainerInfo().getContainerID() + "," + + createdContainers.get(4).getContainerInfo().getContainerID(); //CLI options are passed Optional<Double> threshold = Optional.of(0.1); Optional<Integer> maxDatanodesPercentageToInvolvePerIteration = Optional.of(100); - Optional<Long> maxSizeToMovePerIterationInGB = Optional.of(1L); + Optional<Long> maxSizeToMovePerIterationInGB = Optional.of(6L); Optional<Long> maxSizeEnteringTargetInGB = Optional.of(6L); Optional<Long> maxSizeLeavingSourceInGB = Optional.of(6L); Optional<Integer> moveTimeout = Optional.of(65); @@ -212,7 +226,7 @@ public void testStopBalancerIdempotent() throws IOException { Optional<Integer> iterations = Optional.of(10000); Optional<Integer> maxDatanodesPercentageToInvolvePerIteration = Optional.of(100); - Optional<Long> maxSizeToMovePerIterationInGB = Optional.of(1L); + Optional<Long> maxSizeToMovePerIterationInGB = Optional.of(6L); Optional<Long> maxSizeEnteringTargetInGB = Optional.of(6L); Optional<Long> maxSizeLeavingSourceInGB = Optional.of(6L); Optional<Integer> balancingInterval = Optional.of(70);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDelegationToken.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDelegationToken.java index 1092517..444b6c3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDelegationToken.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDelegationToken.java
@@ -35,12 +35,14 @@ import static org.apache.hadoop.net.ServerSocketUtil.getPort; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY; +import static org.apache.hadoop.ozone.om.OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_AUTH_METHOD; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.TOKEN_ERROR_OTHER; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.TOKEN_EXPIRED; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND; import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS; import static org.assertj.core.api.Assertions.assertThat; @@ -262,7 +264,9 @@ private void initSCM() throws IOException { * 3. Client can authenticate using token. * 4. Delegation token renewal without Kerberos auth fails. * 5. Test success of token cancellation. - * 5. Test failure of token cancellation. + * 6. Test failure of token cancellation. + * 7. Test delegation token renewal failures (expired maxDate, non-matching + * renewer, tampered token). */ @ParameterizedTest @MethodSource("options") @@ -277,6 +281,10 @@ public void testDelegationToken(boolean useIp) throws Exception { GenericTestUtils.setLogLevel(Server.class, INFO); SecurityUtil.setTokenServiceUseIp(useIp); + // Generous token lifetime so the renewal-failure cases below do not race + // token expiry. + conf.setLong(DELEGATION_TOKEN_MAX_LIFETIME_KEY, 60 * 1000L); + // Setup secure OM for start setupOm(conf); @@ -398,12 +406,68 @@ public void testDelegationToken(boolean useIp) throws Exception { assertEquals(TOKEN_ERROR_OTHER, ex.getResult()); assertThat(ex.getMessage()).contains("Cancel delegation token failed"); assertThat(logs.getOutput()).contains("Auth failed for"); + + // Case 7: Delegation token renewal failures. + assertDelegationTokenRenewalFailures(ugi, omLogs); } finally { om.stop(); om.join(); } } + /** + * Exercises the delegation token renewal failure paths against the running + * OM. Reconnects via Kerberos and seeds a fresh token, then verifies renewal + * fails for an expired maxDate, a non-matching renewer, and a tampered token. + */ + private void assertDelegationTokenRenewalFailures( + UserGroupInformation ugi, LogCapturer omLogs) throws Exception { + omClient.close(); + UserGroupInformation.setLoginUser(ugi); + omClient = new OzoneManagerProtocolClientSideTranslatorPB( + OmTransportFactory.create(conf, ugi, null), + RandomStringUtils.secure().nextAscii(5)); + Token<OzoneTokenIdentifier> seedToken = + omClient.getDelegationToken(new Text("om")); + + // 1. When token maxExpiryTime exceeds (maxDate in the past). + OzoneTokenIdentifier expiredId = OzoneTokenIdentifier.readProtoBuf( + seedToken.getIdentifier()); + expiredId.setMaxDate(System.currentTimeMillis() - 1000); + Token<OzoneTokenIdentifier> expiredToken = new Token<>( + expiredId.getBytes(), seedToken.getPassword(), seedToken.getKind(), + seedToken.getService()); + OMException ex = assertThrows(OMException.class, + () -> omClient.renewDelegationToken(expiredToken)); + assertEquals(TOKEN_EXPIRED, ex.getResult()); + omLogs.clearOutput(); + + // 2. When renewer doesn't match (implicitly covers when renewer is + // null or empty). + Token<OzoneTokenIdentifier> token2 = omClient.getDelegationToken( + new Text("randomService")); + assertNotNull(token2); + ex = assertThrows(OMException.class, + () -> omClient.renewDelegationToken(token2)); + assertThat(ex).hasMessageContaining("Delegation token renewal failed"); + assertThat(omLogs.getOutput()).contains(" with non-matching renewer randomService"); + omLogs.clearOutput(); + + // 3. Tampered token that can't be found in cache. + OzoneTokenIdentifier tokenId = OzoneTokenIdentifier.readProtoBuf( + seedToken.getIdentifier()); + tokenId.setRenewer(new Text("om")); + tokenId.setMaxDate(System.currentTimeMillis() * 2); + Token<OzoneTokenIdentifier> tamperedToken = new Token<>( + tokenId.getBytes(), token2.getPassword(), token2.getKind(), + token2.getService()); + ex = assertThrows(OMException.class, + () -> omClient.renewDelegationToken(tamperedToken)); + assertThat(ex).hasMessageContaining("Delegation token renewal failed"); + assertThat(omLogs.getOutput()).contains("can't be found in cache"); + omLogs.clearOutput(); + } + private void generateKeyPair() throws Exception { SecurityConfig securityConfig = new SecurityConfig(conf); HDDSKeyGenerator keyGenerator = new HDDSKeyGenerator(securityConfig);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOMSortDatanodes.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOMSortDatanodes.java index cfce524..a206276 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOMSortDatanodes.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOMSortDatanodes.java
@@ -22,6 +22,7 @@ import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT_LEVEL; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.mockito.Mockito.mock; import com.google.common.collect.ImmutableMap; @@ -176,6 +177,57 @@ private static void assertRackOrder(String rack, List<? extends DatanodeDetails> } } + @Test + public void sortDatanodesForWriteSortsRpcDeserializedPipeline() { + // Pipeline nodes arrive from SCM over RPC as deserialized DatanodeDetails + // with no topology linkage; the write sort must resolve them to OM's cluster + // map, otherwise every node is equidistant (MAX) and the order is random. + List<DatanodeDetails> rpcNodes = new ArrayList<>(); + for (DatanodeDetails dn : nodeManager.getAllNodes()) { + rpcNodes.add(DatanodeDetails.getFromProtoBuf(dn.getProtoBufMessage())); + } + for (DatanodeDetails dn : nodeManager.getAllNodes()) { + // The client address is normally an IP, but the sort must resolve a client + // by either IP or hostname, so cover both. + List<? extends DatanodeDetails> byIp = + keyManager.sortDatanodesForWrite(rpcNodes, dn.getIpAddress(), om.getClusterMap()); + assertEquals(dn, byIp.get(0), + "Source node should be sorted first for writes (IP client)"); + assertRackOrder(dn.getNetworkLocation(), byIp); + + List<? extends DatanodeDetails> byHostname = + keyManager.sortDatanodesForWrite(rpcNodes, dn.getHostName(), om.getClusterMap()); + assertEquals(dn, byHostname.get(0), + "Source node should be sorted first for writes (hostname client)"); + assertRackOrder(dn.getNetworkLocation(), byHostname); + } + } + + @Test + public void sortDatanodesForWriteKeepsOrderForStaleTopology() { + List<DatanodeDetails> nodes = new ArrayList<>(); + nodes.add(randomDatanodeDetails()); + nodes.addAll(nodeManager.getAllNodes()); + + List<? extends DatanodeDetails> sorted = + keyManager.sortDatanodesForWrite(nodes, "edge0", om.getClusterMap()); + + assertSame(nodes, sorted, + "Pipeline order should be preserved when a node is missing from the OM topology"); + } + + @Test + public void sortDatanodesForWriteKeepsOrderWhenClientUnresolved() { + List<? extends DatanodeDetails> nodes = nodeManager.getAllNodes(); + List<DatanodeDetails> original = new ArrayList<>(nodes); + // A client that resolves to no known rack must NOT trigger a shuffle. + String unresolved = nodes.get(0).getIpAddress() + "X"; + List<? extends DatanodeDetails> result = + keyManager.sortDatanodesForWrite(nodes, unresolved, om.getClusterMap()); + assertEquals(original, result, + "Write pipeline order must be preserved when client is unresolved"); + } + private String nodeAddress(DatanodeDetails dn) { boolean useHostname = config.getBoolean( HddsConfigKeys.HDDS_DATANODE_USE_DN_HOSTNAME,
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java index 386ba7c..bb9fb9f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -44,7 +44,6 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY; import static org.apache.hadoop.ozone.OzoneConsts.SCM_SUB_CA; -import static org.apache.hadoop.ozone.om.OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY; @@ -52,7 +51,6 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_S3_GPRC_SERVER_ENABLED; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_TRANSPORT_CLASS; -import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.TOKEN_EXPIRED; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.USER_MISMATCH; import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS; import static org.apache.ozone.test.GenericTestUtils.PortAllocator.getFreePort; @@ -102,12 +100,10 @@ import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto; import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.scm.HddsTestUtils; -import org.apache.hadoop.hdds.scm.ScmConfig; import org.apache.hadoop.hdds.scm.ScmInfo; import org.apache.hadoop.hdds.scm.client.ScmTopologyClient; import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; -import org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.security.SecurityConfig; @@ -128,7 +124,6 @@ import org.apache.hadoop.hdds.utils.HAUtils; import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc_.Client; -import org.apache.hadoop.ipc_.Server; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.ozone.client.OzoneClient; @@ -163,7 +158,9 @@ import org.apache.ozone.test.tag.Unhealthy; import org.apache.ratis.protocol.ClientId; import org.apache.ratis.util.ExitUtils; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -177,38 +174,64 @@ final class TestSecureOzoneCluster { private static final String COMPONENT = "om"; private static final String OM_CERT_SERIAL_ID = "9879877970576"; + private static final String CANNOT_AUTHENTICATE_MESSAGE = "Client cannot authenticate via:[KERBEROS]"; private static final Logger LOG = LoggerFactory .getLogger(TestSecureOzoneCluster.class); + private static final int CERT_GRACE_TIME_MS = 10 * 1000; // 10s + private static final int DELEGATION_TOKEN_MAX_TIME_MS = 9 * 1000; // 9s + + @TempDir + private static File workDir; + private static MiniKdc miniKdc; + private static OzoneConfiguration kdcConf; + private static File scmKeytab; + private static File spnegoKeytab; + private static File omKeytab; + private static File testUserKeytab; + private static String testUserPrincipal; + private static String host; + private static String realm; @TempDir private File tempDir; - - private MiniKdc miniKdc; private OzoneConfiguration conf; - private File workDir; - private File scmKeytab; - private File spnegoKeytab; - private File omKeyTab; - private File testUserKeytab; - private String testUserPrincipal; private StorageContainerManager scm; private ScmBlockLocationProtocol scmBlockClient; private OzoneManager om; private HddsProtos.OzoneManagerDetailsProto omInfo; - private String host; private String clusterId; private String scmId; private String omId; private OzoneManagerProtocolClientSideTranslatorPB omClient; private KeyPair keyPair; private Path omMetaDirPath; - private int certGraceTime = 10 * 1000; // 10s - private int delegationTokenMaxTime = 9 * 1000; // 9s + + @BeforeAll + static void setupKdc() throws Exception { + ExitUtils.disableSystemExit(); + DefaultMetricsSystem.setMiniClusterMode(true); + host = InetAddress.getLocalHost().getCanonicalHostName().toLowerCase(); + startMiniKdc(); + realm = miniKdc.getRealm(); + testUserPrincipal = "test@" + realm; + scmKeytab = new File(workDir, "scm.keytab"); + spnegoKeytab = new File(workDir, "http.keytab"); + omKeytab = new File(workDir, "om.keytab"); + testUserKeytab = new File(workDir, "testuser.keytab"); + kdcConf = new OzoneConfiguration(); + setSecureConfig(kdcConf); + createCredentialsInKDC(kdcConf); + } + + @AfterAll + static void tearDownKdc() { + stopMiniKdc(); + } @BeforeEach void init() { try { - conf = new OzoneConfiguration(); + conf = new OzoneConfiguration(kdcConf); conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "localhost"); conf.setInt(OZONE_SCM_CLIENT_PORT_KEY, getFreePort()); @@ -220,34 +243,25 @@ void init() { conf.set(OZONE_OM_ADDRESS_KEY, InetAddress.getLocalHost().getCanonicalHostName() + ":" + getFreePort()); - DefaultMetricsSystem.setMiniClusterMode(true); - ExitUtils.disableSystemExit(); final String path = tempDir.getAbsolutePath(); omMetaDirPath = Paths.get(path, "om-meta"); conf.set(OZONE_METADATA_DIRS, omMetaDirPath.toString()); - conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true); - conf.set(HADOOP_SECURITY_AUTHENTICATION, KERBEROS.name()); - conf.set(HDDS_X509_CA_ROTATION_CHECK_INTERNAL, - Duration.ofMillis(certGraceTime - 1000).toString()); - conf.set(HDDS_X509_RENEW_GRACE_DURATION, - Duration.ofMillis(certGraceTime).toString()); - conf.setBoolean(HDDS_X509_GRACE_DURATION_TOKEN_CHECKS_ENABLED, false); - conf.set(HDDS_X509_CA_ROTATION_CHECK_INTERNAL, - Duration.ofMillis(certGraceTime - 1000).toString()); - conf.set(HDDS_X509_CA_ROTATION_ACK_TIMEOUT, - Duration.ofMillis(certGraceTime - 1000).toString()); - conf.setLong(OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY, - delegationTokenMaxTime); - workDir = new File(tempDir, "workdir"); + conf.set(HDDS_X509_CA_ROTATION_CHECK_INTERNAL, + Duration.ofMillis(CERT_GRACE_TIME_MS - 1000).toString()); + conf.set(HDDS_X509_RENEW_GRACE_DURATION, + Duration.ofMillis(CERT_GRACE_TIME_MS).toString()); + conf.setBoolean(HDDS_X509_GRACE_DURATION_TOKEN_CHECKS_ENABLED, false); + conf.set(HDDS_X509_CA_ROTATION_ACK_TIMEOUT, + Duration.ofMillis(CERT_GRACE_TIME_MS - 1000).toString()); + conf.setLong(OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY, + DELEGATION_TOKEN_MAX_TIME_MS); + clusterId = UUID.randomUUID().toString(); scmId = UUID.randomUUID().toString(); omId = UUID.randomUUID().toString(); scmBlockClient = new ScmBlockLocationTestingClient(null, null, 0); - startMiniKdc(); - setSecureConfig(); - createCredentialsInKDC(); generateKeyPair(); omInfo = OzoneManager.getOmDetailsProto(conf, omId); } catch (Exception e) { @@ -258,184 +272,247 @@ void init() { @AfterEach void stop() throws Exception { try { - stopMiniKdc(); if (scm != null) { scm.stop(); scm.join(); } - if (om != null) { - om.stop(); - om.join(); - } - IOUtils.closeQuietly(omClient); } catch (Exception e) { LOG.error("Failed to stop TestSecureOzoneCluster", e); + } finally { + IOUtils.closeQuietly(om); + IOUtils.closeQuietly(omClient); } } - private void createCredentialsInKDC() throws Exception { - ScmConfig scmConfig = conf.getObject(ScmConfig.class); - SCMHTTPServerConfig httpServerConfig = - conf.getObject(SCMHTTPServerConfig.class); - createPrincipal(scmKeytab, scmConfig.getKerberosPrincipal()); - createPrincipal(spnegoKeytab, httpServerConfig.getKerberosPrincipal()); + private static void createCredentialsInKDC(OzoneConfiguration conf) throws Exception { + createPrincipal(scmKeytab, conf.get(HDDS_SCM_KERBEROS_PRINCIPAL_KEY)); + createPrincipal(spnegoKeytab, conf.get(HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY)); + createPrincipal(omKeytab, conf.get(OZONE_OM_KERBEROS_PRINCIPAL_KEY)); createPrincipal(testUserKeytab, testUserPrincipal); - createPrincipal(omKeyTab, - conf.get(OZONE_OM_KERBEROS_PRINCIPAL_KEY)); } - private void createPrincipal(File keytab, String... principal) + private static void createPrincipal(File keytab, String... principal) throws Exception { miniKdc.createPrincipal(keytab, principal); } - private void startMiniKdc() throws Exception { + private static void startMiniKdc() throws Exception { Properties securityProperties = MiniKdc.createConf(); miniKdc = new MiniKdc(securityProperties, workDir); miniKdc.start(); } - private void stopMiniKdc() { - miniKdc.stop(); + private static void stopMiniKdc() { + if (miniKdc != null) { + miniKdc.stop(); + } } - private void setSecureConfig() throws IOException { + private static void setSecureConfig(OzoneConfiguration conf) throws IOException { conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true); - host = InetAddress.getLocalHost().getCanonicalHostName() - .toLowerCase(); - conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos"); String curUser = UserGroupInformation.getCurrentUser().getUserName(); conf.set(OZONE_ADMINISTRATORS, curUser); - String realm = miniKdc.getRealm(); String hostAndRealm = host + "@" + realm; conf.set(HDDS_SCM_KERBEROS_PRINCIPAL_KEY, "scm/" + hostAndRealm); conf.set(HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY, "HTTP_SCM/" + hostAndRealm); conf.set(OZONE_OM_KERBEROS_PRINCIPAL_KEY, "om/" + hostAndRealm); conf.set(OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY, "HTTP_OM/" + hostAndRealm); - scmKeytab = new File(workDir, "scm.keytab"); - spnegoKeytab = new File(workDir, "http.keytab"); - omKeyTab = new File(workDir, "om.keytab"); - testUserKeytab = new File(workDir, "testuser.keytab"); - testUserPrincipal = "test@" + realm; - - conf.set(HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY, - scmKeytab.getAbsolutePath()); - conf.set(HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY, - spnegoKeytab.getAbsolutePath()); - conf.set(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY, - omKeyTab.getAbsolutePath()); - conf.set(OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE, - spnegoKeytab.getAbsolutePath()); + conf.set(HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY, scmKeytab.getAbsolutePath()); + conf.set(HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY, spnegoKeytab.getAbsolutePath()); + conf.set(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY, omKeytab.getAbsolutePath()); + conf.set(OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE, spnegoKeytab.getAbsolutePath()); } + /** + * Exercises a secure SCM and OM sharing a single SCM instance to avoid paying + * the SCM startup cost twice. Covers SCM startup and cert trust chain, SCM + * security/admin access control, and secure OM initialization (the + * delegation-token/secret-key config-validation failure, the login success + * case, the Kerberos failure case, and re-initialization of an + * already-initialized OM). + */ @Test - void testSecureScmStartupSuccess() throws Exception { - + void testSecureScmAndOmStartupAndAccessControl() throws Exception { initSCM(); scm = HddsTestUtils.getScmSimple(conf); - //Reads the SCM Info from SCM instance - try { - scm.start(); - ScmInfo scmInfo = scm.getClientProtocolServer().getScmInfo(); - assertEquals(clusterId, scmInfo.getClusterId()); - assertEquals(scmId, scmInfo.getScmId()); - assertEquals(2, scm.getScmCertificateClient().getTrustChain().size()); - } finally { - if (scm != null) { - scm.stop(); - } + scm.start(); + + // SCM startup and access control. + assertScmExposesClusterInfoAndTrustChain(); + assertScmSecurityProtocolAllowsKerberosUser(); + assertScmSecurityProtocolRejectsNonKerberosUser(); + assertScmAdminProtocolDeniesNonAdminUser(); + assertScmAdminProtocolRejectsNonKerberosUser(); + + // Secure OM initialization against the shared SCM. + assertSecureOmInitFailsWithInvalidTokenAndSecretKeyConfig(); + assertSecureOmInitSucceeds(); + assertSecureOmInitFailsWithNonExistentPrincipal(); + assertSecureOmInitWhenAlreadyInitialized(); + } + + /** SCM starts up and exposes its cluster info and cert trust chain. */ + private void assertScmExposesClusterInfoAndTrustChain() throws IOException { + ScmInfo scmInfo = scm.getClientProtocolServer().getScmInfo(); + assertEquals(clusterId, scmInfo.getClusterId()); + assertEquals(scmId, scmInfo.getScmId()); + assertEquals(2, scm.getScmCertificateClient().getTrustChain().size()); + } + + /** SCM security protocol - user with Kerberos credentials succeeds. */ + private void assertScmSecurityProtocolAllowsKerberosUser() throws Exception { + UserGroupInformation ugi = + UserGroupInformation.loginUserFromKeytabAndReturnUGI( + testUserPrincipal, testUserKeytab.getCanonicalPath()); + ugi.setAuthenticationMethod(KERBEROS); + try (SCMSecurityProtocolClientSideTranslatorPB securityClient = + getScmSecurityClient(conf, ugi)) { + assertNotNull(securityClient); + String caCert = securityClient.getCACertificate(); + assertNotNull(caCert); + // Get some random certificate, used serial id 100 which will be + // unavailable as our serial id is time stamp. Serial id 1 is root CA, + // and it is persisted in DB. + SCMSecurityException securityException = assertThrows( + SCMSecurityException.class, + () -> securityClient.getCertificate("100")); + assertThat(securityException) + .hasMessageContaining("Certificate not found"); } } - @Test - void testSCMSecurityProtocol() throws Exception { - - initSCM(); - scm = HddsTestUtils.getScmSimple(conf); - //Reads the SCM Info from SCM instance - try { - scm.start(); - - // Case 1: User with Kerberos credentials should succeed. - UserGroupInformation ugi = - UserGroupInformation.loginUserFromKeytabAndReturnUGI( - testUserPrincipal, testUserKeytab.getCanonicalPath()); - ugi.setAuthenticationMethod(KERBEROS); - try (SCMSecurityProtocolClientSideTranslatorPB securityClient = - getScmSecurityClient(conf, ugi)) { - assertNotNull(securityClient); - String caCert = securityClient.getCACertificate(); - assertNotNull(caCert); - // Get some random certificate, used serial id 100 which will be - // unavailable as our serial id is time stamp. Serial id 1 is root CA, - // and it is persisted in DB. - SCMSecurityException securityException = assertThrows( - SCMSecurityException.class, - () -> securityClient.getCertificate("100")); - assertThat(securityException) - .hasMessageContaining("Certificate not found"); - } - - // Case 2: User without Kerberos credentials should fail. - ugi = UserGroupInformation.createRemoteUser("test"); - ugi.setAuthenticationMethod(AuthMethod.TOKEN); - try (SCMSecurityProtocolClientSideTranslatorPB securityClient = - getScmSecurityClient(conf, ugi)) { - - String cannotAuthMessage = "Client cannot authenticate via:[KERBEROS]"; - IOException ioException = assertThrows(IOException.class, - securityClient::getCACertificate); - assertThat(ioException).hasMessageContaining(cannotAuthMessage); - ioException = assertThrows(IOException.class, - () -> securityClient.getCertificate("1")); - assertThat(ioException).hasMessageContaining(cannotAuthMessage); - } - } finally { - if (scm != null) { - scm.stop(); - } - } - } - - @Test - void testAdminAccessControlException() throws Exception { - initSCM(); - scm = HddsTestUtils.getScmSimple(conf); - //Reads the SCM Info from SCM instance - try { - scm.start(); - - //case 1: Run admin command with non-admin user. - UserGroupInformation ugi = - UserGroupInformation.loginUserFromKeytabAndReturnUGI( - testUserPrincipal, testUserKeytab.getCanonicalPath()); - StorageContainerLocationProtocol scmRpcClient = - HAUtils.getScmContainerClient(conf, ugi); + /** SCM security protocol - user without Kerberos credentials fails. */ + private void assertScmSecurityProtocolRejectsNonKerberosUser() throws Exception { + UserGroupInformation ugi = UserGroupInformation.createRemoteUser("test"); + ugi.setAuthenticationMethod(AuthMethod.TOKEN); + try (SCMSecurityProtocolClientSideTranslatorPB securityClient = + getScmSecurityClient(conf, ugi)) { IOException ioException = assertThrows(IOException.class, - scmRpcClient::forceExitSafeMode); - assertThat(ioException).hasMessageContaining("Access denied"); - - // Case 2: User without Kerberos credentials should fail. - ugi = UserGroupInformation.createRemoteUser("test"); - ugi.setAuthenticationMethod(AuthMethod.TOKEN); - scmRpcClient = - HAUtils.getScmContainerClient(conf, ugi); - - String cannotAuthMessage = "Client cannot authenticate via:[KERBEROS]"; + securityClient::getCACertificate); + assertThat(ioException).hasMessageContaining(CANNOT_AUTHENTICATE_MESSAGE); ioException = assertThrows(IOException.class, - scmRpcClient::forceExitSafeMode); - assertThat(ioException).hasMessageContaining(cannotAuthMessage); - } finally { - if (scm != null) { - scm.stop(); - } + () -> securityClient.getCertificate("1")); + assertThat(ioException).hasMessageContaining(CANNOT_AUTHENTICATE_MESSAGE); } } + /** SCM admin protocol - authenticated non-admin user is denied. */ + private void assertScmAdminProtocolDeniesNonAdminUser() throws IOException { + UserGroupInformation ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI( + testUserPrincipal, testUserKeytab.getCanonicalPath()); + StorageContainerLocationProtocol scmRpcClient = + HAUtils.getScmContainerClient(conf, ugi); + IOException adminException = assertThrows(IOException.class, + scmRpcClient::forceExitSafeMode); + assertThat(adminException).hasMessageContaining("Access denied"); + } + + /** SCM admin protocol - user without Kerberos credentials fails. */ + private void assertScmAdminProtocolRejectsNonKerberosUser() throws IOException { + UserGroupInformation ugi = UserGroupInformation.createRemoteUser("test"); + ugi.setAuthenticationMethod(AuthMethod.TOKEN); + StorageContainerLocationProtocol scmRpcClient = + HAUtils.getScmContainerClient(conf, ugi); + IOException adminException = assertThrows(IOException.class, + scmRpcClient::forceExitSafeMode); + assertThat(adminException) + .hasMessageContaining(CANNOT_AUTHENTICATE_MESSAGE); + } + + /** + * Secure OM initialization fails when the delegation token and secret key + * configuration don't meet requirement. + */ + private void assertSecureOmInitFailsWithInvalidTokenAndSecretKeyConfig() { + conf.setTimeDuration(HDDS_SECRET_KEY_EXPIRY_DURATION, 7, TimeUnit.DAYS); + conf.setTimeDuration(OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY, 7, TimeUnit.DAYS); + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, () -> setupOm(conf)); + assertThat(exception.getMessage()).contains("Secret key expiry duration hdds.secret.key.expiry.duration " + + "should be greater than value of (ozone.manager.delegation.token.max-lifetime + " + + "ozone.manager.delegation.remover.scan.interval + hdds.secret.key.rotate.duration"); + // Restore valid durations so the remaining cases can start OM. + conf.unset(HDDS_SECRET_KEY_EXPIRY_DURATION); + conf.setLong(OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY, DELEGATION_TOKEN_MAX_TIME_MS); + } + + /** Secure OM initialization succeeds. */ + private void assertSecureOmInitSucceeds() throws Exception { + LogCapturer logs = LogCapturer.captureLogs(OzoneManager.class); + GenericTestUtils.setLogLevel(OzoneManager.class, INFO); + setupOm(conf); + assertThrows(Exception.class, om::start); + assertThat(logs.getOutput()).contains("Ozone Manager login successful"); + logs.clearOutput(); + stopOm(); + } + + /** + * Secure OM initialization fails at Kerberos login for a non-existent + * principal. Storage is already initialized by + * {@link #assertSecureOmInitSucceeds()}, and createOm with a non-existent + * principal fails at Kerberos login before any storage check or RPC bind, so + * no additional setup is required. + */ + private void assertSecureOmInitFailsWithNonExistentPrincipal() { + conf.set(OZONE_OM_KERBEROS_PRINCIPAL_KEY, + "non-existent-user@EXAMPLE.com"); + testCommonKerberosFailures(() -> OzoneManager.createOm(conf)); + } + + /** + * Secure OM can be re-initialized when it is already initialized. + * The failure cases above left a non-existent principal and an invalid auth + * method on conf; restore the valid values before re-initializing. A fresh + * RPC port is also required: {@link #assertSecureOmInitSucceeds()}'s start() + * failed before starting the RPC server, so its listener selector loop never + * ran and stop() cannot release the bound port. + */ + private void assertSecureOmInitWhenAlreadyInitialized() throws Exception { + conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + conf.set(OZONE_OM_KERBEROS_PRINCIPAL_KEY, "om/" + host + "@" + realm); + conf.set(OZONE_OM_ADDRESS_KEY, + InetAddress.getLocalHost().getCanonicalHostName() + ":" + getFreePort()); + LogCapturer omLogs = LogCapturer.captureLogs(OMCertificateClient.class); + omLogs.clearOutput(); + conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, false); + OMStorage omStore = new OMStorage(conf); + initializeOmStorage(omStore); + OzoneManager.setTestSecureOmFlag(true); + om = OzoneManager.createOm(conf); + + assertNull(om.getCertificateClient()); + String logOutput = omLogs.getOutput(); + assertThat(logOutput) + .doesNotContain("Init response: GETCERT"); + assertThat(logOutput) + .doesNotContain("Successfully stored SCM signed certificate"); + + stopOm(); + + conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true); + conf.setBoolean(OZONE_OM_S3_GPRC_SERVER_ENABLED, true); + conf.set(OZONE_OM_ADDRESS_KEY, + InetAddress.getLocalHost().getCanonicalHostName() + ":" + getFreePort()); + + OzoneManager.omInit(conf); + om = OzoneManager.createOm(conf); + + assertNotNull(om.getCertificateClient()); + assertNotNull(om.getCertificateClient().getPublicKey()); + assertNotNull(om.getCertificateClient().getPrivateKey()); + assertNotNull(om.getCertificateClient().getCertificate()); + assertThat(omLogs.getOutput()) + .contains("Init response: GETCERT") + .contains("Successfully stored OM signed certificate"); + X509Certificate certificate = om.getCertificateClient().getCertificate(); + validateCertificate(certificate); + } + private void initSCM() throws IOException { Path scmPath = new File(tempDir, "scm-meta").toPath(); Files.createDirectories(scmPath); @@ -504,97 +581,19 @@ private void testCommonKerberosFailures(Callable<?> test) { .hasMessageContaining("KERBEROS_SSL authentication method not"); } - /** - * Tests the secure om Initialization Failure. - */ - @Test - void testSecureOMInitializationFailure() throws Exception { - initSCM(); - // Create a secure SCM instance as om client will connect to it - scm = HddsTestUtils.getScmSimple(conf); - try { - scm.start(); - setupOm(conf); - conf.set(OZONE_OM_KERBEROS_PRINCIPAL_KEY, - "non-existent-user@EXAMPLE.com"); - testCommonKerberosFailures(() -> OzoneManager.createOm(conf)); - } finally { - if (scm != null) { - scm.stop(); - } - } - } - - /** - * Tests the secure om Initialization Failure due to delegation token and secret key configuration don't meet - * requirement. - */ - @Test - void testSecureOMDelegationTokenSecretManagerInitializationFailure() throws Exception { - initSCM(); - // Create a secure SCM instance as om client will connect to it - scm = HddsTestUtils.getScmSimple(conf); - try { - scm.start(); - conf.setTimeDuration(HDDS_SECRET_KEY_EXPIRY_DURATION, 7, TimeUnit.DAYS); - conf.setTimeDuration(OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY, 7, TimeUnit.DAYS); - IllegalArgumentException exception = assertThrows( - IllegalArgumentException.class, () -> setupOm(conf)); - assertThat(exception.getMessage()).contains("Secret key expiry duration hdds.secret.key.expiry.duration " + - "should be greater than value of (ozone.manager.delegation.token.max-lifetime + " + - "ozone.manager.delegation.remover.scan.interval + hdds.secret.key.rotate.duration"); - } finally { - if (scm != null) { - scm.stop(); - } - } - } - - /** - * Tests the secure om Initialization success. - */ - @Test - void testSecureOmInitializationSuccess() throws Exception { - initSCM(); - // Create a secure SCM instance as om client will connect to it - scm = HddsTestUtils.getScmSimple(conf); - LogCapturer logs = LogCapturer.captureLogs(OzoneManager.class); - GenericTestUtils.setLogLevel(OzoneManager.class, INFO); - - try { - scm.start(); - setupOm(conf); - om.start(); - } catch (Exception ex) { - // Expects timeout failure from scmClient in om but om user login via - // kerberos should succeed. - assertThat(logs.getOutput()).contains("Ozone Manager login successful"); - } finally { - if (scm != null) { - scm.stop(); - } - } - } - @Test void testAccessControlExceptionOnClient() throws Exception { initSCM(); - LogCapturer logs = LogCapturer.captureLogs(OzoneManager.class); - GenericTestUtils.setLogLevel(OzoneManager.class, INFO); - try { - // Create a secure SCM instance as om client will connect to it - scm = HddsTestUtils.getScmSimple(conf); - scm.start(); + // Create a secure SCM instance as om client will connect to it + scm = HddsTestUtils.getScmSimple(conf); + scm.start(); - setupOm(conf); - om.setCertClient(new CertificateClientTestImpl(conf)); - om.setScmTopologyClient(new ScmTopologyClient(scmBlockClient)); - om.start(); - } catch (Exception ex) { - // Expects timeout failure from scmClient in om but om user login via - // kerberos should succeed. - assertThat(logs.getOutput()).contains("Ozone Manager login successful"); - } + setupOm(conf); + om.setCertClient(new CertificateClientTestImpl(conf)); + om.setScmTopologyClient(new ScmTopologyClient(scmBlockClient)); + om.start(); + + // positive case (happy-path) UserGroupInformation ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI( testUserPrincipal, testUserKeytab.getCanonicalPath()); @@ -609,6 +608,7 @@ void testAccessControlExceptionOnClient() throws Exception { .setAdminName("admin") .build()); + // negative (an unauthenticated client gets rejected) ugi = UserGroupInformation.createUserForTesting( "testuser1", new String[] {"test"}); @@ -618,7 +618,7 @@ void testAccessControlExceptionOnClient() throws Exception { ClientId.randomId().toString()); String exMessage = "org.apache.hadoop.security.AccessControlException: " + "Client cannot authenticate via:[TOKEN, KERBEROS]"; - logs = LogCapturer.captureLogs(Client.class); + LogCapturer logs = LogCapturer.captureLogs(Client.class); IOException ioException = assertThrows(IOException.class, () -> unsecureClient.listAllVolumes(null, null, 0)); assertThat(ioException).hasMessageContaining(exMessage); @@ -634,304 +634,145 @@ private void generateKeyPair() throws Exception { keyStorage.storeKeyPair(keyPair); } - /** - * Tests delegation token renewal. - */ - @Test - void testDelegationTokenRenewal() throws Exception { - GenericTestUtils.setLogLevel(Server.class, INFO); - LogCapturer omLogs = LogCapturer.captureLogs(OzoneManager.class); - - // Setup SCM - initSCM(); - scm = HddsTestUtils.getScmSimple(conf); - try { - // Start SCM - scm.start(); - - // Setup secure OM for start. Generous token lifetime so the renewer and - // tampered-token cases below do not race token expiry. - conf.setLong(DELEGATION_TOKEN_MAX_LIFETIME_KEY, 60 * 1000L); - setupOm(conf); - OzoneManager.setTestSecureOmFlag(true); - om.setCertClient(new CertificateClientTestImpl(conf)); - om.setScmTopologyClient(new ScmTopologyClient(scmBlockClient)); - om.start(); - - UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - - // Get first OM client which will authenticate via Kerberos - omClient = new OzoneManagerProtocolClientSideTranslatorPB( - OmTransportFactory.create(conf, ugi, null), - RandomStringUtils.secure().nextAscii(5)); - - // Since client is already connected get a delegation token - Token<OzoneTokenIdentifier> token = omClient.getDelegationToken( - new Text("om")); - - // Check if token is of right kind and renewer is running om instance - assertNotNull(token); - assertEquals("OzoneToken", token.getKind().toString()); - assertEquals(SecurityUtil.buildTokenService( - om.getNodeDetails().getRpcAddress()).toString(), - token.getService().toString()); - - // Renew delegation token - long expiryTime = omClient.renewDelegationToken(token); - assertThat(expiryTime).isGreaterThan(0); - omLogs.clearOutput(); - - // Test failure of delegation renewal - // 1. When token maxExpiryTime exceeds (maxDate in the past) - OzoneTokenIdentifier expiredId = OzoneTokenIdentifier.readProtoBuf( - token.getIdentifier()); - expiredId.setMaxDate(System.currentTimeMillis() - 1000); - Token<OzoneTokenIdentifier> expiredToken = new Token<>( - expiredId.getBytes(), token.getPassword(), token.getKind(), - token.getService()); - OMException ex = assertThrows(OMException.class, - () -> omClient.renewDelegationToken(expiredToken)); - assertEquals(TOKEN_EXPIRED, ex.getResult()); - omLogs.clearOutput(); - - // 2. When renewer doesn't match (implicitly covers when renewer is - // null or empty ) - Token<OzoneTokenIdentifier> token2 = omClient.getDelegationToken( - new Text("randomService")); - assertNotNull(token2); - ex = assertThrows(OMException.class, - () -> omClient.renewDelegationToken(token2)); - assertThat(ex).hasMessageContaining("Delegation token renewal failed"); - assertThat(omLogs.getOutput()).contains(" with non-matching renewer randomService"); - omLogs.clearOutput(); - - // 3. Test tampered token - OzoneTokenIdentifier tokenId = OzoneTokenIdentifier.readProtoBuf( - token.getIdentifier()); - tokenId.setRenewer(new Text("om")); - tokenId.setMaxDate(System.currentTimeMillis() * 2); - Token<OzoneTokenIdentifier> tamperedToken = new Token<>( - tokenId.getBytes(), token2.getPassword(), token2.getKind(), - token2.getService()); - ex = assertThrows(OMException.class, - () -> omClient.renewDelegationToken(tamperedToken)); - assertThat(ex).hasMessageContaining("Delegation token renewal failed"); - assertThat(omLogs.getOutput()).contains("can't be found in cache"); - omLogs.clearOutput(); - } finally { - if (scm != null) { - scm.stop(); - } - IOUtils.closeQuietly(om); - } - } - private void setupOm(OzoneConfiguration config) throws Exception { OMStorage omStore = new OMStorage(config); - omStore.setClusterId(clusterId); - omStore.setOmCertSerialId(OM_CERT_SERIAL_ID); - // writes the version file properties - omStore.initialize(); + if (omStore.getState() != Storage.StorageState.INITIALIZED) { + omStore.setClusterId(clusterId); + omStore.setOmCertSerialId(OM_CERT_SERIAL_ID); + // writes the version file properties + omStore.initialize(); + } OzoneManager.setTestSecureOmFlag(true); om = OzoneManager.createOm(config); } + private void stopOm() { + if (om != null) { + if (om.stop()) { + om.join(); + } + om = null; + } + } + @Test @Flaky("HDDS-9349") void testGetSetRevokeS3Secret() throws Exception { initSCM(); - try { - scm = HddsTestUtils.getScmSimple(conf); - scm.start(); + scm = HddsTestUtils.getScmSimple(conf); + scm.start(); - // Setup secure OM for start - setupOm(conf); - // Start OM - om.setCertClient(new CertificateClientTestImpl(conf)); - om.setScmTopologyClient(new ScmTopologyClient(scmBlockClient)); - om.start(); - UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - String username = ugi.getUserName(); + // Setup secure OM for start + setupOm(conf); + // Start OM + om.setCertClient(new CertificateClientTestImpl(conf)); + om.setScmTopologyClient(new ScmTopologyClient(scmBlockClient)); + om.start(); + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + String username = ugi.getUserName(); - // Get first OM client which will authenticate via Kerberos - omClient = new OzoneManagerProtocolClientSideTranslatorPB( - OmTransportFactory.create(conf, ugi, null), - RandomStringUtils.secure().nextAscii(5)); + // Get first OM client which will authenticate via Kerberos + omClient = new OzoneManagerProtocolClientSideTranslatorPB( + OmTransportFactory.create(conf, ugi, null), + RandomStringUtils.secure().nextAscii(5)); - // Creates a secret since it does not exist - S3SecretValue attempt1 = omClient.getS3Secret(username); + // Creates a secret since it does not exist + S3SecretValue attempt1 = omClient.getS3Secret(username); - // A second getS3Secret on the same username should throw exception - try { - omClient.getS3Secret(username); - } catch (OMException omEx) { - assertEquals(OMException.ResultCodes.S3_SECRET_ALREADY_EXISTS, - omEx.getResult()); - } + // A second getS3Secret on the same username should throw exception + OMException omEx = assertThrows(OMException.class, + () -> omClient.getS3Secret(username)); + assertEquals(OMException.ResultCodes.S3_SECRET_ALREADY_EXISTS, + omEx.getResult()); - // Revoke the existing secret - omClient.revokeS3Secret(username); + // Revoke the existing secret + omClient.revokeS3Secret(username); - // Set secret should fail since the accessId is revoked - final String secretKeySet = "somesecret1"; - try { - omClient.setS3Secret(username, secretKeySet); - } catch (OMException omEx) { - assertEquals(OMException.ResultCodes.ACCESS_ID_NOT_FOUND, - omEx.getResult()); - } + // Set secret should fail since the accessId is revoked + final String secretKeySet = "somesecret1"; + omEx = assertThrows(OMException.class, + () -> omClient.setS3Secret(username, secretKeySet)); + assertEquals(OMException.ResultCodes.ACCESS_ID_NOT_FOUND, + omEx.getResult()); - // Get a new secret - S3SecretValue attempt3 = omClient.getS3Secret(username); + // Get a new secret + S3SecretValue attempt3 = omClient.getS3Secret(username); - // secret should differ because it has been revoked previously - assertNotEquals(attempt3.getAwsSecret(), attempt1.getAwsSecret()); + // secret should differ because it has been revoked previously + assertNotEquals(attempt3.getAwsSecret(), attempt1.getAwsSecret()); - // accessKey is still the same because it is derived from username - assertEquals(attempt3.getAwsAccessKey(), attempt1.getAwsAccessKey()); + // accessKey is still the same because it is derived from username + assertEquals(attempt3.getAwsAccessKey(), attempt1.getAwsAccessKey()); - // Admin can set secret for any user - S3SecretValue attempt4 = omClient.setS3Secret(username, secretKeySet); - assertEquals(secretKeySet, attempt4.getAwsSecret()); + // Admin can set secret for any user + S3SecretValue attempt4 = omClient.setS3Secret(username, secretKeySet); + assertEquals(secretKeySet, attempt4.getAwsSecret()); - // A second getS3Secret on the same username should throw exception - try { - omClient.getS3Secret(username); - } catch (OMException omEx) { - assertEquals(OMException.ResultCodes.S3_SECRET_ALREADY_EXISTS, - omEx.getResult()); - } + // A second getS3Secret on the same username should throw exception + omEx = assertThrows(OMException.class, + () -> omClient.getS3Secret(username)); + assertEquals(OMException.ResultCodes.S3_SECRET_ALREADY_EXISTS, + omEx.getResult()); - // Clean up - omClient.revokeS3Secret(username); + // Clean up + omClient.revokeS3Secret(username); - // Admin can get and revoke other users' secrets - // omClient's ugi is current user, which is added as an OM admin - omClient.getS3Secret("HADOOP/ALICE"); - omClient.revokeS3Secret("HADOOP/ALICE"); + // Admin can get and revoke other users' secrets + // omClient's ugi is current user, which is added as an OM admin + omClient.getS3Secret("HADOOP/ALICE"); + omClient.revokeS3Secret("HADOOP/ALICE"); - // testUser is not an admin - final UserGroupInformation ugiNonAdmin = - UserGroupInformation.loginUserFromKeytabAndReturnUGI( - testUserPrincipal, testUserKeytab.getCanonicalPath()); - final OzoneManagerProtocolClientSideTranslatorPB omClientNonAdmin = - new OzoneManagerProtocolClientSideTranslatorPB( - OmTransportFactory.create(conf, ugiNonAdmin, null), - RandomStringUtils.secure().nextAscii(5)); + // testUser is not an admin + final UserGroupInformation ugiNonAdmin = + UserGroupInformation.loginUserFromKeytabAndReturnUGI( + testUserPrincipal, testUserKeytab.getCanonicalPath()); + final OzoneManagerProtocolClientSideTranslatorPB omClientNonAdmin = + new OzoneManagerProtocolClientSideTranslatorPB( + OmTransportFactory.create(conf, ugiNonAdmin, null), + RandomStringUtils.secure().nextAscii(5)); - OMException omException = assertThrows(OMException.class, - () -> omClientNonAdmin.getS3Secret("HADOOP/JOHN")); - assertSame(USER_MISMATCH, omException.getResult()); - omException = assertThrows(OMException.class, - () -> omClientNonAdmin.revokeS3Secret("HADOOP/DOE")); - assertSame(USER_MISMATCH, omException.getResult()); - - } finally { - if (scm != null) { - scm.stop(); - } - IOUtils.closeQuietly(om); - } - } - - /** - * Tests functionality to init secure OM when it is already initialized. - */ - @Test - void testSecureOmReInit() throws Exception { - LogCapturer omLogs = LogCapturer.captureLogs(OMCertificateClient.class); - omLogs.clearOutput(); - - initSCM(); - try { - scm = HddsTestUtils.getScmSimple(conf); - scm.start(); - conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, false); - OMStorage omStore = new OMStorage(conf); - initializeOmStorage(omStore); - OzoneManager.setTestSecureOmFlag(true); - om = OzoneManager.createOm(conf); - - assertNull(om.getCertificateClient()); - String logOutput = omLogs.getOutput(); - assertThat(logOutput) - .doesNotContain("Init response: GETCERT"); - assertThat(logOutput) - .doesNotContain("Successfully stored SCM signed certificate"); - - if (om.stop()) { - om.join(); - } - - conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true); - conf.setBoolean(OZONE_OM_S3_GPRC_SERVER_ENABLED, true); - conf.set(OZONE_OM_ADDRESS_KEY, - InetAddress.getLocalHost().getCanonicalHostName() + ":" + getFreePort()); - - OzoneManager.omInit(conf); - om = OzoneManager.createOm(conf); - - assertNotNull(om.getCertificateClient()); - assertNotNull(om.getCertificateClient().getPublicKey()); - assertNotNull(om.getCertificateClient().getPrivateKey()); - assertNotNull(om.getCertificateClient().getCertificate()); - assertThat(omLogs.getOutput()) - .contains("Init response: GETCERT") - .contains("Successfully stored OM signed certificate"); - X509Certificate certificate = om.getCertificateClient().getCertificate(); - validateCertificate(certificate); - - } finally { - if (scm != null) { - scm.stop(); - } - } + OMException omException = assertThrows(OMException.class, + () -> omClientNonAdmin.getS3Secret("HADOOP/JOHN")); + assertSame(USER_MISMATCH, omException.getResult()); + omException = assertThrows(OMException.class, + () -> omClientNonAdmin.revokeS3Secret("HADOOP/DOE")); + assertSame(USER_MISMATCH, omException.getResult()); } /** * Test functionality to get SCM signed certificate for OM. */ @Test - void testSecureOmInitSuccess() throws Exception { + void testSecureOmGetsScmSignedCertificate() throws Exception { LogCapturer omLogs = LogCapturer.captureLogs(OMCertificateClient.class); omLogs.clearOutput(); initSCM(); - try { - scm = HddsTestUtils.getScmSimple(conf); - scm.start(); + scm = HddsTestUtils.getScmSimple(conf); + scm.start(); - OMStorage omStore = new OMStorage(conf); - initializeOmStorage(omStore); - OzoneManager.setTestSecureOmFlag(true); - om = OzoneManager.createOm(conf); + OMStorage omStore = new OMStorage(conf); + initializeOmStorage(omStore); + OzoneManager.setTestSecureOmFlag(true); + om = OzoneManager.createOm(conf); - assertNotNull(om.getCertificateClient()); - assertNotNull(om.getCertificateClient().getPublicKey()); - assertNotNull(om.getCertificateClient().getPrivateKey()); - assertNotNull(om.getCertificateClient().getCertificate()); - assertEquals(3, om.getCertificateClient().getTrustChain().size()); - assertThat(omLogs.getOutput()) - .contains("Init response: GETCERT") - .contains("Successfully stored OM signed certificate"); - X509Certificate certificate = om.getCertificateClient().getCertificate(); - validateCertificate(certificate); - String pemEncodedCACert = - scm.getSecurityProtocolServer().getCACertificate(); - X509Certificate caCert = - CertificateCodec.getX509Certificate(pemEncodedCACert); - X509Certificate caCertStored = om.getCertificateClient() - .getCertificate(caCert.getSerialNumber().toString()); - assertEquals(caCert, caCertStored); - } finally { - if (scm != null) { - scm.stop(); - } - if (om != null) { - om.stop(); - } - IOUtils.closeQuietly(om); - } + assertNotNull(om.getCertificateClient()); + assertNotNull(om.getCertificateClient().getPublicKey()); + assertNotNull(om.getCertificateClient().getPrivateKey()); + assertNotNull(om.getCertificateClient().getCertificate()); + assertEquals(3, om.getCertificateClient().getTrustChain().size()); + assertThat(omLogs.getOutput()) + .contains("Init response: GETCERT") + .contains("Successfully stored OM signed certificate"); + X509Certificate certificate = om.getCertificateClient().getCertificate(); + validateCertificate(certificate); + String pemEncodedCACert = + scm.getSecurityProtocolServer().getCACertificate(); + X509Certificate caCert = + CertificateCodec.getX509Certificate(pemEncodedCACert); + X509Certificate caCertStored = om.getCertificateClient() + .getCertificate(caCert.getSerialNumber().toString()); + assertEquals(caCert, caCertStored); } /** @@ -1112,7 +953,6 @@ void testCertificateRotationUnRecoverableFailure() throws Exception { LogCapturer certClientLogs = LogCapturer.captureLogs(OMCertificateClient.class); LogCapturer exitUtilLog = LogCapturer.captureLogs(ExitUtil.class); - OMStorage omStorage = new OMStorage(conf); omStorage.setClusterId(clusterId); omStorage.setOmId(omId); @@ -1172,77 +1012,70 @@ public String renewAndStoreKeyAndCertificate(boolean force) throws CertificateEx @Test void testDelegationTokenRenewCrossSecretKeyRotation() throws Exception { initSCM(); - try { - scm = HddsTestUtils.getScmSimple(conf); - scm.start(); + scm = HddsTestUtils.getScmSimple(conf); + scm.start(); - // Setup secure OM for start. - final int certLifetime = 40 * 1000; // 40s - OzoneConfiguration newConf = new OzoneConfiguration(conf); - newConf.set(HDDS_X509_DEFAULT_DURATION, - Duration.ofMillis(certLifetime).toString()); - newConf.set(HDDS_X509_RENEW_GRACE_DURATION, - Duration.ofMillis(certLifetime - 15 * 1000).toString()); - newConf.setLong(OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY, - certLifetime - 20 * 1000); + // Setup secure OM for start. + final int certLifetime = 40 * 1000; // 40s + OzoneConfiguration newConf = new OzoneConfiguration(conf); + newConf.set(HDDS_X509_DEFAULT_DURATION, + Duration.ofMillis(certLifetime).toString()); + newConf.set(HDDS_X509_RENEW_GRACE_DURATION, + Duration.ofMillis(certLifetime - 15 * 1000).toString()); + newConf.setLong(OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY, + certLifetime - 20 * 1000); - setupOm(newConf); - OzoneManager.setTestSecureOmFlag(true); + setupOm(newConf); + OzoneManager.setTestSecureOmFlag(true); - CertificateClientTestImpl certClient = - new CertificateClientTestImpl(newConf, true); - // Start OM - om.setCertClient(certClient); - om.setScmTopologyClient(new ScmTopologyClient(scmBlockClient)); - SecretKeyTestClient secretKeyClient = new SecretKeyTestClient(); - ManagedSecretKey secretKey1 = secretKeyClient.getCurrentSecretKey(); - om.setSecretKeyClient(secretKeyClient); - om.start(); - GenericTestUtils.waitFor(() -> om.isLeaderReady(), 100, 10000); + CertificateClientTestImpl certClient = + new CertificateClientTestImpl(newConf, true); + // Start OM + om.setCertClient(certClient); + om.setScmTopologyClient(new ScmTopologyClient(scmBlockClient)); + SecretKeyTestClient secretKeyClient = new SecretKeyTestClient(); + ManagedSecretKey secretKey1 = secretKeyClient.getCurrentSecretKey(); + om.setSecretKeyClient(secretKeyClient); + om.start(); + GenericTestUtils.waitFor(() -> om.isLeaderReady(), 100, 10000); - UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - // Get first OM client which will authenticate via Kerberos - omClient = new OzoneManagerProtocolClientSideTranslatorPB( - OmTransportFactory.create(newConf, ugi, null), - RandomStringUtils.secure().nextAscii(5)); + // Get first OM client which will authenticate via Kerberos + omClient = new OzoneManagerProtocolClientSideTranslatorPB( + OmTransportFactory.create(newConf, ugi, null), + RandomStringUtils.secure().nextAscii(5)); - // Since client is already connected get a delegation token - Token<OzoneTokenIdentifier> token1 = omClient.getDelegationToken( - new Text("om")); + // Since client is already connected get a delegation token + Token<OzoneTokenIdentifier> token1 = omClient.getDelegationToken( + new Text("om")); - // Check if token is of right kind and renewer is running om instance - assertNotNull(token1); - assertEquals("OzoneToken", token1.getKind().toString()); - assertEquals(SecurityUtil.buildTokenService( - om.getNodeDetails().getRpcAddress()).toString(), - token1.getService().toString()); - assertEquals(secretKey1.getId().toString(), token1.decodeIdentifier().getSecretKeyId()); + // Check if token is of right kind and renewer is running om instance + assertNotNull(token1); + assertEquals("OzoneToken", token1.getKind().toString()); + assertEquals(SecurityUtil.buildTokenService( + om.getNodeDetails().getRpcAddress()).toString(), + token1.getService().toString()); + assertEquals(secretKey1.getId().toString(), token1.decodeIdentifier().getSecretKeyId()); - // Renew delegation token - long expiryTime = omClient.renewDelegationToken(token1); - assertThat(expiryTime).isGreaterThan(0); + // Renew delegation token + long expiryTime = omClient.renewDelegationToken(token1); + assertThat(expiryTime).isGreaterThan(0); - // Rotate secret key - secretKeyClient.rotate(); - ManagedSecretKey secretKey2 = secretKeyClient.getCurrentSecretKey(); - assertNotEquals(secretKey1.getId(), secretKey2.getId()); - // Get a new delegation token - Token<OzoneTokenIdentifier> token2 = omClient.getDelegationToken( - new Text("om")); - assertEquals(secretKey2.getId().toString(), token2.decodeIdentifier().getSecretKeyId()); + // Rotate secret key + secretKeyClient.rotate(); + ManagedSecretKey secretKey2 = secretKeyClient.getCurrentSecretKey(); + assertNotEquals(secretKey1.getId(), secretKey2.getId()); + // Get a new delegation token + Token<OzoneTokenIdentifier> token2 = omClient.getDelegationToken( + new Text("om")); + assertEquals(secretKey2.getId().toString(), token2.decodeIdentifier().getSecretKeyId()); - // Because old secret key is still valid, so renew old token will succeed - expiryTime = omClient.renewDelegationToken(token1); - assertThat(expiryTime) - .isGreaterThan(0) - .isLessThan(secretKey2.getExpiryTime().toEpochMilli()); - } finally { - if (scm != null) { - scm.stop(); - } - IOUtils.closeQuietly(om); - } + // Because old secret key is still valid, so renew old token will succeed + expiryTime = omClient.renewDelegationToken(token1); + assertThat(expiryTime) + .isGreaterThan(0) + .isLessThan(secretKey2.getExpiryTime().toEpochMilli()); } /**
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/admin/om/lease/TestLeaseRecoverer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/admin/om/lease/TestLeaseRecoverer.java index a93add3..71e8eb6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/admin/om/lease/TestLeaseRecoverer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/admin/om/lease/TestLeaseRecoverer.java
@@ -35,7 +35,7 @@ import org.apache.hadoop.fs.LeaseRecoverable; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.ozone.TestDataUtil; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -59,7 +59,7 @@ void init() throws Exception { client = cluster().newClient(); // create a volume and a FSO bucket - fsoOzoneBucket = TestDataUtil + fsoOzoneBucket = DataTestUtil .createVolumeAndBucket(client, BucketLayout.FILE_SYSTEM_OPTIMIZED); }
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java index f64d91f..a356201 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java
@@ -138,6 +138,7 @@ import org.apache.hadoop.hdds.utils.FaultInjector; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.ClientConfigForTesting; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OmUtils; @@ -146,7 +147,6 @@ import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.OzoneTestUtils; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.audit.AuditLogTestUtils; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.ObjectStore; @@ -1116,19 +1116,19 @@ public void testDeleteAuditLog() throws Exception { // create a three replica file String keyName1 = "key1"; - TestDataUtil.createKey(bucket, keyName1, ReplicationConfig + DataTestUtil.createKey(bucket, keyName1, ReplicationConfig .fromTypeAndFactor(RATIS, THREE), value); // create a EC replica file String keyName2 = "key2"; ReplicationConfig replicationConfig = new ECReplicationConfig("rs-3-2-1024k"); - TestDataUtil.createKey(bucket, keyName2, replicationConfig, value); + DataTestUtil.createKey(bucket, keyName2, replicationConfig, value); // create a directory and a file String dirName = "dir1"; bucket.createDirectory(dirName); String keyName3 = "key3"; - TestDataUtil.createKey(bucket, keyName3, ReplicationConfig + DataTestUtil.createKey(bucket, keyName3, ReplicationConfig .fromTypeAndFactor(RATIS, THREE), value); // delete files and directory @@ -1139,11 +1139,11 @@ public void testDeleteAuditLog() throws Exception { // create keys for deleteKeys case String keyName4 = "key4"; - TestDataUtil.createKey(bucket, dirName + "/" + keyName4, + DataTestUtil.createKey(bucket, dirName + "/" + keyName4, ReplicationConfig.fromTypeAndFactor(RATIS, THREE), value); String keyName5 = "key5"; - TestDataUtil.createKey(bucket, dirName + "/" + keyName5, replicationConfig, value); + DataTestUtil.createKey(bucket, dirName + "/" + keyName5, replicationConfig, value); List<String> keysToDelete = new ArrayList<>(); keysToDelete.add(dirName + "/" + keyName4); @@ -1264,7 +1264,7 @@ public void testPutKeyWithReplicationConfig(String replicationValue, ReplicationConfig replicationConfig = new ECReplicationConfig(replicationValue); if (isValidReplicationConfig) { - TestDataUtil.createKey(bucket, keyName, + DataTestUtil.createKey(bucket, keyName, replicationConfig, value.getBytes(UTF_8)); OzoneKey key = bucket.getKey(keyName); assertEquals(keyName, key.getName()); @@ -1295,7 +1295,7 @@ public void testPutKey() throws IOException { for (int i = 0; i < 10; i++) { String keyName = UUID.randomUUID().toString(); - TestDataUtil.createKey(bucket, keyName, + DataTestUtil.createKey(bucket, keyName, ReplicationConfig.fromTypeAndFactor(RATIS, ONE), value.getBytes(UTF_8)); OzoneKey key = bucket.getKey(keyName); @@ -2211,7 +2211,7 @@ public void testValidateBlockLengthWithCommitKey() throws IOException { String keyName = UUID.randomUUID().toString(); // create the initial key with size 0, write will allocate the first block. - TestDataUtil.createKey(bucket, keyName, + DataTestUtil.createKey(bucket, keyName, ReplicationConfig.fromTypeAndFactor(RATIS, ONE), value.getBytes(UTF_8)); OmKeyArgs.Builder builder = new OmKeyArgs.Builder(); @@ -2245,7 +2245,7 @@ public void testPutKeyRatisOneNode() throws IOException { for (int i = 0; i < 10; i++) { String keyName = UUID.randomUUID().toString(); - TestDataUtil.createKey(bucket, keyName, + DataTestUtil.createKey(bucket, keyName, ReplicationConfig.fromTypeAndFactor(RATIS, ONE), value.getBytes(UTF_8)); OzoneKey key = bucket.getKey(keyName); @@ -2278,7 +2278,7 @@ public void testPutKeyRatisThreeNodes() throws IOException { for (int i = 0; i < 10; i++) { String keyName = UUID.randomUUID().toString(); - TestDataUtil.createKey(bucket, keyName, + DataTestUtil.createKey(bucket, keyName, ReplicationConfig.fromTypeAndFactor(RATIS, THREE), value.getBytes(UTF_8)); OzoneKey key = bucket.getKey(keyName); @@ -2317,7 +2317,7 @@ public void testPutKeyRatisThreeNodesParallel() throws IOException, String keyName = UUID.randomUUID().toString(); String data = Arrays.toString(generateData(5 * 1024 * 1024, (byte) RandomUtils.secure().randomLong())); - TestDataUtil.createKey(bucket, keyName, + DataTestUtil.createKey(bucket, keyName, ReplicationConfig.fromTypeAndFactor(RATIS, THREE), data.getBytes(UTF_8)); OzoneKey key = bucket.getKey(keyName); @@ -2391,7 +2391,7 @@ private void createAndCorruptKey(String volumeName, String bucketName, OzoneBucket bucket = volume.getBucket(bucketName); // Write data into a key - TestDataUtil.createKey(bucket, keyName, + DataTestUtil.createKey(bucket, keyName, ReplicationConfig.fromTypeAndFactor(RATIS, ONE), value.getBytes(UTF_8)); @@ -2451,7 +2451,7 @@ public void testGetKeyDetails() throws IOException { String keyValue = RandomStringUtils.secure().next(128); //String keyValue = "this is a test value.glx"; // create the initial key with size 0, write will allocate the first block. - TestDataUtil.createKey(bucket, keyName, + DataTestUtil.createKey(bucket, keyName, ReplicationConfig.fromTypeAndFactor(RATIS, ONE), keyValue.getBytes(UTF_8)); @@ -2544,7 +2544,7 @@ public void testReadKeyWithCorruptedData() throws IOException { String keyName = UUID.randomUUID().toString(); // Write data into a key - TestDataUtil.createKey(bucket, keyName, + DataTestUtil.createKey(bucket, keyName, ReplicationConfig.fromTypeAndFactor(RATIS, ONE), value.getBytes(UTF_8)); @@ -2593,14 +2593,14 @@ void testZReadKeyWithUnhealthyContainerReplica() throws Exception { String keyName1 = UUID.randomUUID().toString(); // Write first key - TestDataUtil.createKey(bucket, keyName1, + DataTestUtil.createKey(bucket, keyName1, ReplicationConfig.fromTypeAndFactor(RATIS, THREE), value.getBytes(UTF_8)); // Write second key String keyName2 = UUID.randomUUID().toString(); value = "unhealthy container replica"; - TestDataUtil.createKey(bucket, keyName2, + DataTestUtil.createKey(bucket, keyName2, ReplicationConfig.fromTypeAndFactor(RATIS, THREE), value.getBytes(UTF_8)); @@ -2686,7 +2686,7 @@ void testReadKeyWithCorruptedDataWithMutiNodes() throws IOException { String keyName = UUID.randomUUID().toString(); // Write data into a key - TestDataUtil.createKey(bucket, keyName, + DataTestUtil.createKey(bucket, keyName, ReplicationConfig.fromTypeAndFactor(RATIS, THREE), value.getBytes(UTF_8)); @@ -2753,7 +2753,7 @@ public void testDeleteKey() OzoneVolume volume = store.getVolume(volumeName); volume.createBucket(bucketName); OzoneBucket bucket = volume.getBucket(bucketName); - TestDataUtil.createKey(bucket, keyName, + DataTestUtil.createKey(bucket, keyName, ReplicationConfig.fromTypeAndFactor(RATIS, ONE), value.getBytes(UTF_8)); OzoneKey key = bucket.getKey(keyName); @@ -3044,16 +3044,16 @@ public void testListKey() String keyBaseA = "key-a-"; for (int i = 0; i < 10; i++) { byte[] value = RandomStringUtils.secure().nextAscii(10240).getBytes(UTF_8); - TestDataUtil.createKey(volAbucketA, + DataTestUtil.createKey(volAbucketA, keyBaseA + i + "-" + RandomStringUtils.secure().nextNumeric(5), ReplicationConfig.fromTypeAndFactor(RATIS, ONE), value); - TestDataUtil.createKey(volAbucketB, + DataTestUtil.createKey(volAbucketB, keyBaseA + i + "-" + RandomStringUtils.secure().nextNumeric(5), ReplicationConfig.fromTypeAndFactor(RATIS, ONE), value); - TestDataUtil.createKey(volBbucketA, + DataTestUtil.createKey(volBbucketA, keyBaseA + i + "-" + RandomStringUtils.secure().nextNumeric(5), ReplicationConfig.fromTypeAndFactor(RATIS, ONE), value); - TestDataUtil.createKey(volBbucketB, + DataTestUtil.createKey(volBbucketB, keyBaseA + i + "-" + RandomStringUtils.secure().nextNumeric(5), ReplicationConfig.fromTypeAndFactor(RATIS, ONE), value); } @@ -3065,16 +3065,16 @@ public void testListKey() String keyBaseB = "key-b-"; for (int i = 0; i < 10; i++) { byte[] value = RandomStringUtils.secure().nextAscii(10240).getBytes(UTF_8); - TestDataUtil.createKey(volAbucketA, + DataTestUtil.createKey(volAbucketA, keyBaseB + i + "-" + RandomStringUtils.secure().nextNumeric(5), ReplicationConfig.fromTypeAndFactor(RATIS, ONE), value); - TestDataUtil.createKey(volAbucketB, + DataTestUtil.createKey(volAbucketB, keyBaseB + i + "-" + RandomStringUtils.secure().nextNumeric(5), ReplicationConfig.fromTypeAndFactor(RATIS, ONE), value); - TestDataUtil.createKey(volBbucketA, + DataTestUtil.createKey(volBbucketA, keyBaseB + i + "-" + RandomStringUtils.secure().nextNumeric(5), ReplicationConfig.fromTypeAndFactor(RATIS, ONE), value); - TestDataUtil.createKey(volBbucketB, + DataTestUtil.createKey(volBbucketB, keyBaseB + i + "-" + RandomStringUtils.secure().nextNumeric(5), ReplicationConfig.fromTypeAndFactor(RATIS, ONE), value); } @@ -3555,7 +3555,7 @@ public void testClientLeakDetector() throws Exception { OzoneBucket bucket = volume.getBucket(bucketName); byte[] data = new byte[10]; Arrays.fill(data, (byte) 1); - TestDataUtil.createKey(bucket, keyName, + DataTestUtil.createKey(bucket, keyName, ReplicationConfig.fromTypeAndFactor(RATIS, ONE), data); client = null; @@ -4601,7 +4601,7 @@ private void validateOzoneAccessAcl(OzoneObj ozObj) throws IOException { } private void writeKey(String key1, OzoneBucket bucket) throws IOException { - TestDataUtil.createKey(bucket, key1, + DataTestUtil.createKey(bucket, key1, ReplicationConfig.fromTypeAndFactor(RATIS, ONE), RandomStringUtils.secure().next(1024).getBytes(UTF_8)); } @@ -5004,7 +5004,7 @@ public void testHeadObject() throws IOException { String keyName = UUID.randomUUID().toString(); - TestDataUtil.createKey(bucket, keyName, + DataTestUtil.createKey(bucket, keyName, replicationConfig, value.getBytes(UTF_8)); OzoneKey key = bucket.headObject(keyName); @@ -5043,11 +5043,11 @@ private void createRequiredForVersioningTest(String volumeName, .setBucketLayout(VERSIONING_TEST_BUCKET_LAYOUT).build()); OzoneBucket bucket = volume.getBucket(bucketName); - TestDataUtil.createKey(bucket, keyName, + DataTestUtil.createKey(bucket, keyName, replicationConfig, value.getBytes(UTF_8)); // Override key - TestDataUtil.createKey(bucket, keyName, + DataTestUtil.createKey(bucket, keyName, replicationConfig, value.getBytes(UTF_8)); } @@ -5266,7 +5266,7 @@ void testGetKeyAndFileWithNetworkTopology() throws IOException { String keyName = UUID.randomUUID().toString(); // Write data into a key - TestDataUtil.createKey(bucket, keyName, + DataTestUtil.createKey(bucket, keyName, ReplicationConfig.fromTypeAndFactor(RATIS, THREE), value.getBytes(UTF_8)); @@ -5550,7 +5550,7 @@ public void testPutObjectTagging(BucketLayout bucketLayout) throws Exception { String keyName = UUID.randomUUID().toString(); - TestDataUtil.createKey(bucket, keyName, + DataTestUtil.createKey(bucket, keyName, anyReplication(), value.getBytes(UTF_8)); OzoneKey key = bucket.getKey(keyName);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBCSID.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBCSID.java index b046323..feed906 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBCSID.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBCSID.java
@@ -35,8 +35,8 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; @@ -91,7 +91,7 @@ public static void shutdown() { @Test public void testBCSID() throws Exception { - TestDataUtil.createKey(objectStore.getVolume(volumeName).getBucket(bucketName), + DataTestUtil.createKey(objectStore.getVolume(volumeName).getBucket(bucketName), "ratis", ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), "ratis".getBytes(UTF_8));
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java index 3cea059..b4337a4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
@@ -152,8 +152,10 @@ static MiniOzoneCluster createCluster() throws IOException, private static Stream<Arguments> clientParameters() { return Stream.of( - Arguments.of(true), - Arguments.of(false) + Arguments.of(true, true), + Arguments.of(true, false), + Arguments.of(false, true), + Arguments.of(false, false) ); } @@ -166,11 +168,19 @@ private static Stream<Arguments> dataLengthParameters() { ); } + private static Stream<Arguments> streamWriteParameters() { + return dataLengthParameters().flatMap(dataLength -> + Stream.of(true, false).map(putBlockOnCloseEnabled -> + Arguments.of(dataLength.get()[0], putBlockOnCloseEnabled))); + } + static OzoneClientConfig newClientConfig(ConfigurationSource source, - boolean flushDelay) { + boolean flushDelay, + boolean putBlockOnCloseEnabled) { OzoneClientConfig clientConfig = source.getObject(OzoneClientConfig.class); clientConfig.setChecksumType(ContainerProtos.ChecksumType.NONE); clientConfig.setStreamBufferFlushDelay(flushDelay); + clientConfig.setDatastreamPutBlockOnCloseEnabled(putBlockOnCloseEnabled); return clientConfig; } @@ -198,13 +208,17 @@ public void shutdown() { } @ParameterizedTest - @MethodSource("dataLengthParameters") + @MethodSource("streamWriteParameters") @Flaky("HDDS-12027") - public void testStreamWrite(int dataLength) throws Exception { - OzoneClientConfig config = newClientConfig(cluster.getConf(), false); + public void testStreamWrite(int dataLength, boolean putBlockOnCloseEnabled) throws Exception { + OzoneClientConfig config = newClientConfig(cluster.getConf(), false, putBlockOnCloseEnabled); try (OzoneClient client = newClient(cluster.getConf(), config)) { testWrite(client, dataLength); - testWriteWithFailure(client, dataLength); + // Forced container close before stream close relies on async PutBlock recovery; + // that path is not used when PutBlock is committed only on data stream close. + if (!putBlockOnCloseEnabled) { + testWriteWithFailure(client, dataLength); + } } } @@ -253,8 +267,9 @@ static void validateData(OzoneClient client, String keyName, byte[] data) throws @ParameterizedTest @MethodSource("clientParameters") - public void testPutBlockAtBoundary(boolean flushDelay) throws Exception { - OzoneClientConfig config = newClientConfig(cluster.getConf(), flushDelay); + public void testPutBlockAtBoundary(boolean flushDelay, boolean putBlockOnCloseEnabled) + throws Exception { + OzoneClientConfig config = newClientConfig(cluster.getConf(), flushDelay, putBlockOnCloseEnabled); try (OzoneClient client = newClient(cluster.getConf(), config)) { int dataLength = 500; XceiverClientMetrics metrics = @@ -273,19 +288,21 @@ public void testPutBlockAtBoundary(boolean flushDelay) throws Exception { assertThat(metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock)) .isLessThanOrEqualTo(pendingPutBlockCount + 1); key.close(); - // Since data length is 500 , first putBlock will be at 400(flush boundary) - // and the other at 500 + // Since data length is 500, first putBlock will be at 400 (flush boundary). + // Close commits via WriteAsync PutBlock only when putBlockOnClose is disabled. + int expectedPutBlocks = putBlockOnCloseEnabled ? 1 : 2; assertEquals( metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock), - putBlockCount + 2); + putBlockCount + expectedPutBlocks); validateData(client, keyName, data); } } @ParameterizedTest @MethodSource("clientParameters") - public void testMinPacketSize(boolean flushDelay) throws Exception { - OzoneClientConfig config = newClientConfig(cluster.getConf(), flushDelay); + public void testMinPacketSize(boolean flushDelay, boolean putBlockOnCloseEnabled) + throws Exception { + OzoneClientConfig config = newClientConfig(cluster.getConf(), flushDelay, putBlockOnCloseEnabled); try (OzoneClient client = newClient(cluster.getConf(), config)) { String keyName = getKeyName(); XceiverClientMetrics metrics = @@ -312,8 +329,9 @@ public void testMinPacketSize(boolean flushDelay) throws Exception { @ParameterizedTest @MethodSource("clientParameters") - public void testTotalAckDataLength(boolean flushDelay) throws Exception { - OzoneClientConfig config = newClientConfig(cluster.getConf(), flushDelay); + public void testTotalAckDataLength(boolean flushDelay, boolean putBlockOnCloseEnabled) + throws Exception { + OzoneClientConfig config = newClientConfig(cluster.getConf(), flushDelay, putBlockOnCloseEnabled); try (OzoneClient client = newClient(cluster.getConf(), config)) { int dataLength = 400; String keyName = getKeyName(); @@ -334,8 +352,9 @@ public void testTotalAckDataLength(boolean flushDelay) throws Exception { @ParameterizedTest @MethodSource("clientParameters") - public void testDatanodeVersion(boolean flushDelay) throws Exception { - OzoneClientConfig config = newClientConfig(cluster.getConf(), flushDelay); + public void testDatanodeVersion(boolean flushDelay, boolean putBlockOnCloseEnabled) + throws Exception { + OzoneClientConfig config = newClientConfig(cluster.getConf(), flushDelay, putBlockOnCloseEnabled); try (OzoneClient client = newClient(cluster.getConf(), config)) { // Verify all DNs internally have versions set correctly List<HddsDatanodeService> dns = cluster.getHddsDatanodes();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java index 58af0db..8f8bc5c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java
@@ -26,11 +26,15 @@ import java.nio.ByteBuffer; import java.util.List; import java.util.UUID; +import java.util.stream.Stream; import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.client.ReplicationType; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.io.KeyDataStreamOutput; import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput; import org.apache.hadoop.ozone.container.ContainerTestHelper; @@ -40,7 +44,8 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; /** * Tests the containerStateMachine stream handling. @@ -48,7 +53,6 @@ @TestInstance(TestInstance.Lifecycle.PER_CLASS) public abstract class TestContainerStateMachineStream implements NonHATests.TestCase { private OzoneClient client; - private ObjectStore objectStore; private String volumeName; private String bucketName; private int chunkSize; @@ -58,7 +62,7 @@ void setup() throws Exception { chunkSize = (int) cluster().getConf().getStorageSize(OZONE_SCM_CHUNK_SIZE_KEY, 1024 * 1024, StorageUnit.BYTES); client = cluster().newClient(); - objectStore = client.getObjectStore(); + ObjectStore objectStore = client.getObjectStore(); volumeName = "vol-" + UUID.randomUUID(); bucketName = "teststreambucket"; @@ -71,14 +75,26 @@ void shutdown() { IOUtils.closeQuietly(client); } + private static Stream<Arguments> streamingParameters() { + return Stream.of(-1, +1).flatMap(offset -> + Stream.of(false, true).map(putBlockOnCloseEnabled -> + Arguments.of(offset, putBlockOnCloseEnabled))); + } + @ParameterizedTest - @ValueSource(ints = {-1, +1}) - void testContainerStateMachineForStreaming(int offset) throws Exception { + @MethodSource("streamingParameters") + void testContainerStateMachineForStreaming(int offset, boolean putBlockOnCloseEnabled) + throws Exception { final int size = chunkSize + offset; + OzoneConfiguration conf = new OzoneConfiguration(cluster().getConf()); + OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class); + clientConfig.setDatastreamPutBlockOnCloseEnabled(putBlockOnCloseEnabled); + conf.setFromObject(clientConfig); final List<OmKeyLocationInfo> locationInfoList; - try (OzoneDataStreamOutput key = createStreamKey("key" + offset, ReplicationType.RATIS, size, - objectStore, volumeName, bucketName)) { + try (OzoneClient streamingClient = OzoneClientFactory.getRpcClient(conf); + OzoneDataStreamOutput key = createStreamKey("key" + offset + "-" + putBlockOnCloseEnabled, + ReplicationType.RATIS, size, streamingClient.getObjectStore(), volumeName, bucketName)) { byte[] data = ContainerTestHelper.generateData(size, true); key.write(ByteBuffer.wrap(data));
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestHybridPipelineOnDatanode.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestHybridPipelineOnDatanode.java index dd4cb3e..1892029 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestHybridPipelineOnDatanode.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestHybridPipelineOnDatanode.java
@@ -38,8 +38,8 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -96,14 +96,14 @@ public void testHybridPipelineOnDatanode() throws IOException { String keyName1 = UUID.randomUUID().toString(); // Write data into a key - TestDataUtil.createKey(bucket, keyName1, + DataTestUtil.createKey(bucket, keyName1, ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), value.getBytes(UTF_8)); String keyName2 = UUID.randomUUID().toString(); // Write data into a key - TestDataUtil.createKey(bucket, keyName2, + DataTestUtil.createKey(bucket, keyName2, ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), value.getBytes(UTF_8));
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java index 281c5b2..a4fc753 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java
@@ -84,9 +84,9 @@ import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClientTestImpl; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.ClientConfigForTesting; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; @@ -299,7 +299,7 @@ static void createAndVerifyKeyData(OzoneBucket bucket) throws Exception { String keyName = UUID.randomUUID().toString(); String value = "sample value"; - TestDataUtil.createKey(bucket, keyName, + DataTestUtil.createKey(bucket, keyName, ReplicationConfig.fromTypeAndFactor(RATIS, ONE), value.getBytes(StandardCharsets.UTF_8)); @@ -307,7 +307,7 @@ static void createAndVerifyKeyData(OzoneBucket bucket) throws Exception { OzoneKeyDetails key1 = bucket.getKey(keyName); // Overwrite the key - TestDataUtil.createKey(bucket, keyName, + DataTestUtil.createKey(bucket, keyName, ReplicationConfig.fromTypeAndFactor(RATIS, ONE), value.getBytes(StandardCharsets.UTF_8));
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java index 53ff4c1..ff0c690 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java
@@ -82,9 +82,11 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartCommitUploadPartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmMultipartPartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.QuotaUtil; +import org.apache.hadoop.ozone.om.request.util.OMMultipartUploadUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.ozone.test.NonHATests; import org.junit.jupiter.api.AfterAll; @@ -657,14 +659,27 @@ private void verifyPartNamesInDB(Map<Integer, String> partsMap, metadataMgr.getMultipartInfoTable().get(multipartKey); assertNotNull(omMultipartKeyInfo); - for (OzoneManagerProtocolProtos.PartKeyInfo partKeyInfo : - omMultipartKeyInfo.getPartKeyInfoMap()) { - String partKeyName = partKeyInfo.getPartName(); + // Collect the part names as stored in the DB. For the split parts-table + // schema the parts live in the multipart parts table rather than inline + // in the multipart info table. + List<String> dbPartNames = new ArrayList<>(); + if (omMultipartKeyInfo.getSchemaVersion() + == OmMultipartKeyInfo.SPLIT_PARTS_TABLE_SCHEMA_VERSION) { + for (OmMultipartPartInfo partInfo : + OMMultipartUploadUtils.scanParts(metadataMgr, uploadID).values()) { + dbPartNames.add(partInfo.getPartName()); + } + } else { + for (OzoneManagerProtocolProtos.PartKeyInfo partKeyInfo : + omMultipartKeyInfo.getPartKeyInfoMap()) { + dbPartNames.add(partKeyInfo.getPartName()); + } + } - // reconstruct full part name with volume, bucket, partKeyName - String fullKeyPartName = - metadataMgr.getOzoneKey(volumeName, bucketName, keyName); - + // reconstruct full part name with volume, bucket, partKeyName + String fullKeyPartName = + metadataMgr.getOzoneKey(volumeName, bucketName, keyName); + for (String partKeyName : dbPartNames) { // partKeyName format in DB - partKeyName + ClientID assertTrue(partKeyName.startsWith(fullKeyPartName), "Invalid partKeyName format in DB: " + partKeyName
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientWithKeyLatestVersion.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientWithKeyLatestVersion.java index 703b379..03c706f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientWithKeyLatestVersion.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientWithKeyLatestVersion.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.ozone.TestDataUtil; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; @@ -102,7 +102,7 @@ private void createAndOverwriteKey(OzoneBucket bucket, String key, private static void writeKey(OzoneBucket bucket, String key, byte[] content, ReplicationConfig replication) throws IOException { - TestDataUtil.createKey(bucket, key, replication, content); + DataTestUtil.createKey(bucket, key, replication, content); } private void assertListStatus(OzoneBucket bucket, String keyName,
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestReadRetries.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestReadRetries.java index 70989f0..15c9dec 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestReadRetries.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestReadRetries.java
@@ -36,8 +36,8 @@ import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -78,7 +78,7 @@ void testPutKeyAndGetKeyThreeNodes() throws Exception { String keyName = "a/b/c/" + UUID.randomUUID(); byte[] content = RandomUtils.secure().randomBytes(128); - TestDataUtil.createKey(bucket, keyName, + DataTestUtil.createKey(bucket, keyName, RatisReplicationConfig.getInstance(THREE), content); // First, confirm the key info from the client matches the info in OM.
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java index 772ec03..4b53c2d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java
@@ -20,11 +20,11 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS; import static org.apache.hadoop.hdds.security.SecurityConfig.OZONE_TEST_AUTHORIZATION_ENABLED; +import static org.apache.hadoop.ozone.DataTestUtil.cleanupDeletedTable; import static org.apache.hadoop.ozone.OzoneConsts.FORCE_LEASE_RECOVERY_ENV; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_ROOT; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; -import static org.apache.hadoop.ozone.TestDataUtil.cleanupDeletedTable; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; import static org.apache.hadoop.ozone.om.helpers.BucketLayout.FILE_SYSTEM_OPTIMIZED; import static org.apache.ozone.test.GenericTestUtils.getTestStartTime; @@ -59,9 +59,9 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -167,7 +167,7 @@ private void testPutKeySuccessWithBlockTokenWithBucketLayout( String keyName = UUID.randomUUID().toString(); long committedBytes = ozoneManager.getMetrics().getDataCommittedBytes(); - TestDataUtil.createKey(bucket, keyName, replication, value.getBytes(UTF_8)); + DataTestUtil.createKey(bucket, keyName, replication, value.getBytes(UTF_8)); assertEquals(committedBytes + value.getBytes(UTF_8).length, ozoneManager.getMetrics().getDataCommittedBytes());
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestStreamReadDatanodeFailover.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestStreamReadDatanodeFailover.java index 830610a..ebfeb28 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestStreamReadDatanodeFailover.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestStreamReadDatanodeFailover.java
@@ -47,8 +47,8 @@ import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.scm.storage.StreamBlockInputStream; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -120,7 +120,7 @@ void testAsyncReadFailoverWhenDatanodeStopped() throws Exception { OzoneBucket bucket = createBucket(store); String keyName = newKeyName(); byte[] content = RandomUtils.secure().randomBytes(32 * 1024); - TestDataUtil.createKey(bucket, keyName, + DataTestUtil.createKey(bucket, keyName, RatisReplicationConfig.getInstance(THREE), content); List<DatanodeDetails> datanodes = getPipelineDatanodes(cluster, bucket, keyName); @@ -146,7 +146,7 @@ void testStreamReadFailoverWhenDatanodesStopped() throws Exception { OzoneBucket bucket = createBucket(store); String keyName = newKeyName(); byte[] content = RandomUtils.secure().randomBytes(32 * 1024); - TestDataUtil.createKey(bucket, keyName, + DataTestUtil.createKey(bucket, keyName, RatisReplicationConfig.getInstance(THREE), content); List<DatanodeDetails> datanodes = getPipelineDatanodes(cluster, bucket, keyName);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java index 9b1c756..5db2781 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java
@@ -31,7 +31,6 @@ import static org.apache.ozone.test.GenericTestUtils.setLogLevel; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.Mockito.any; @@ -40,12 +39,10 @@ import com.google.common.collect.ImmutableMap; import java.io.IOException; import java.time.Duration; -import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; @@ -65,13 +62,12 @@ import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRandom; import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager.ReplicationManagerConfiguration; import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.container.common.interfaces.Container; @@ -85,7 +81,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -105,12 +100,10 @@ class TestContainerReplication { SCMContainerPlacementRandom.class ); - static List<Arguments> containerReplicationArguments() { - List<Arguments> arguments = new LinkedList<>(); + static List<String> containerReplicationArguments() { + List<String> arguments = new LinkedList<>(); for (Class<? extends PlacementPolicy> policyClass : POLICIES) { - String canonicalName = policyClass.getCanonicalName(); - arguments.add(Arguments.arguments(canonicalName, true)); - arguments.add(Arguments.arguments(canonicalName, false)); + arguments.add(policyClass.getCanonicalName()); } return arguments; } @@ -122,27 +115,30 @@ static void setUp() { setLogLevel(SCMContainerPlacementRandom.class, Level.DEBUG); } + /** + * Verifies that a closed RATIS THREE container which becomes under-replicated + * after a datanode shutdown is restored to three replicas by ReplicationManager, + * and that the configured placement policy records the datanode-choose metrics. + * Runs once per placement policy in {@link #containerReplicationArguments()}. + */ @ParameterizedTest @MethodSource("containerReplicationArguments") - void testContainerReplication( - String placementPolicyClass, boolean legacyEnabled) throws Exception { + void testRatisContainerReReplicationAfterDatanodeShutdown(String placementPolicyClass) throws Exception { - OzoneConfiguration conf = createConfiguration(legacyEnabled); + OzoneConfiguration conf = createConfiguration(); conf.set(OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY, placementPolicyClass); - try (MiniOzoneCluster cluster = newCluster(conf)) { + try (MiniOzoneCluster cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build()) { cluster.waitForClusterToBeReady(); SCMContainerPlacementMetrics metrics = cluster.getStorageContainerManager().getPlacementMetrics(); try (OzoneClient client = cluster.newClient()) { createTestData(client); - List<OmKeyLocationInfo> keyLocations = lookupKey(cluster); - assertThat(keyLocations).isNotEmpty(); long datanodeChooseAttemptCount = metrics.getDatanodeChooseAttemptCount(); long datanodeChooseSuccessCount = metrics.getDatanodeChooseSuccessCount(); long datanodeChooseFallbackCount = metrics.getDatanodeChooseFallbackCount(); long datanodeRequestCount = metrics.getDatanodeRequestCount(); - OmKeyLocationInfo keyLocation = keyLocations.get(0); + OmKeyLocationInfo keyLocation = lookupKeyFirstLocation(cluster); long containerID = keyLocation.getContainerID(); waitForContainerClose(cluster, containerID); @@ -151,7 +147,7 @@ void testContainerReplication( waitForReplicaCount(containerID, 3, cluster); - Supplier<String> messageSupplier = () -> "policy=" + placementPolicyClass + " legacy=" + legacyEnabled; + Supplier<String> messageSupplier = () -> "policy=" + placementPolicyClass; assertEquals(datanodeRequestCount + 1, metrics.getDatanodeRequestCount(), messageSupplier); assertThat(metrics.getDatanodeChooseAttemptCount()).isGreaterThan(datanodeChooseAttemptCount); assertEquals(datanodeChooseSuccessCount + 1, metrics.getDatanodeChooseSuccessCount(), messageSupplier); @@ -160,14 +156,7 @@ void testContainerReplication( } } - private static MiniOzoneCluster newCluster(OzoneConfiguration conf) - throws IOException { - return MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(5) - .build(); - } - - private static OzoneConfiguration createConfiguration(boolean enableLegacy) { + private static OzoneConfiguration createConfiguration() { OzoneConfiguration conf = new OzoneConfiguration(); conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, TimeUnit.SECONDS); conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, TimeUnit.SECONDS); @@ -181,47 +170,23 @@ private static OzoneConfiguration createConfiguration(boolean enableLegacy) { return conf; } - // TODO use common helper to create test data private void createTestData(OzoneClient client) throws IOException { - ObjectStore objectStore = client.getObjectStore(); - objectStore.createVolume(VOLUME); - OzoneVolume volume = objectStore.getVolume(VOLUME); - volume.createBucket(BUCKET); + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client, VOLUME, BUCKET); - OzoneBucket bucket = volume.getBucket(BUCKET); - - TestDataUtil.createKey(bucket, KEY, + DataTestUtil.createKey(bucket, KEY, RatisReplicationConfig.getInstance(THREE), "Hello".getBytes(UTF_8)); } private byte[] createTestData(OzoneClient client, int size) throws IOException { - ObjectStore objectStore = client.getObjectStore(); - objectStore.createVolume(VOLUME); - OzoneVolume volume = objectStore.getVolume(VOLUME); - volume.createBucket(BUCKET); - OzoneBucket bucket = volume.getBucket(BUCKET); + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client, VOLUME, BUCKET); - byte[] b = new byte[size]; - b = RandomUtils.secure().randomBytes(b.length); - TestDataUtil.createKey(bucket, KEY, + byte[] b = RandomUtils.secure().randomBytes(size); + DataTestUtil.createKey(bucket, KEY, new ECReplicationConfig("RS-3-2-1k"), b); return b; } - private static List<OmKeyLocationInfo> lookupKey(MiniOzoneCluster cluster) - throws IOException { - OmKeyArgs keyArgs = new OmKeyArgs.Builder() - .setVolumeName(VOLUME) - .setBucketName(BUCKET) - .setKeyName(KEY) - .build(); - OmKeyInfo keyInfo = cluster.getOzoneManager().lookupKey(keyArgs); - OmKeyLocationInfoGroup locations = keyInfo.getLatestVersionLocations(); - assertNotNull(locations); - return locations.getLocationList(); - } - private static OmKeyLocationInfo lookupKeyFirstLocation(MiniOzoneCluster cluster) throws IOException { OmKeyArgs keyArgs = new OmKeyArgs.Builder() @@ -278,12 +243,12 @@ private static void deleteContainer(MiniOzoneCluster cluster, DatanodeDetails dn @Test public void testImportedContainerIsClosed() throws Exception { - OzoneConfiguration conf = createConfiguration(false); + OzoneConfiguration conf = createConfiguration(); // create a 4 node cluster try (MiniOzoneCluster cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(4).build()) { cluster.waitForClusterToBeReady(); - try (OzoneClient client = OzoneClientFactory.getRpcClient(conf)) { + try (OzoneClient client = cluster.newClient()) { List<DatanodeDetails> allNodes = cluster.getHddsDatanodes().stream() .map(HddsDatanodeService::getDatanodeDetails) @@ -315,9 +280,9 @@ public void testImportedContainerIsClosed() throws Exception { @Test @Flaky("HDDS-11087") public void testECContainerReplication() throws Exception { - OzoneConfiguration conf = createConfiguration(false); + OzoneConfiguration conf = createConfiguration(); final Map<Integer, Integer> failedReadChunkCountMap = new ConcurrentHashMap<>(); - // Overiding Config to support 1k Chunk size + // Overriding Config to support 1k Chunk size conf.set("ozone.replication.allowed-configs", "(^((STANDALONE|RATIS)/(ONE|THREE))|(EC/(3-2|6-3|10-4)-" + "(512|1024|2048|4096|1)k)$)"); conf.set(OZONE_SCM_CONTAINER_PLACEMENT_EC_IMPL_KEY, SCMContainerPlacementRackScatter.class.getCanonicalName()); @@ -327,20 +292,7 @@ public void testECContainerReplication() throws Exception { // Creating Cluster with 5 Nodes try (MiniOzoneCluster cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build()) { cluster.waitForClusterToBeReady(); - try (OzoneClient client = OzoneClientFactory.getRpcClient(conf)) { - Set<DatanodeDetails> allNodes = - cluster.getHddsDatanodes().stream().map(HddsDatanodeService::getDatanodeDetails).collect( - Collectors.toSet()); - List<DatanodeDetails> initialNodesWithData = new ArrayList<>(); - // Keeping 5 DNs and stopping the 6th Node here it is kept in the var extraNodes - for (DatanodeDetails dn : allNodes) { - if (initialNodesWithData.size() < 5) { - initialNodesWithData.add(dn); - } else { - cluster.shutdownHddsDatanode(dn); - } - } - + try (OzoneClient client = cluster.newClient()) { // Creating 2 stripes with Chunk Size 1k int size = 6 * 1024; byte[] originalData = createTestData(client, size); @@ -350,6 +302,12 @@ public void testECContainerReplication() throws Exception { long containerID = keyLocation.getContainerID(); waitForContainerClose(cluster, containerID); + // The cluster has 5 datanodes and the key is written as EC RS-3-2 (3 data + 2 parity = 5 + // replica indices), so every datanode now holds exactly one replica index. + List<DatanodeDetails> initialNodesWithData = + cluster.getHddsDatanodes().stream().map(HddsDatanodeService::getDatanodeDetails) + .collect(Collectors.toList()); + // Forming Replica Index Map Map<Integer, DatanodeDetails> replicaIndexMap = initialNodesWithData.stream().map(dn -> new Object[]{dn, keyLocation.getPipeline().getReplicaIndex(dn)})
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReportHandling.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReportHandling.java index 415b991..287ed05 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReportHandling.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReportHandling.java
@@ -47,8 +47,8 @@ import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -149,7 +149,7 @@ void testDeletingOrDeletedContainerWhenNonEmptyReplicaIsReported() throws Except ReplicationInput replicationInput = testCase.getRight(); // create a container and close it String key = "key" + keyCount; - TestDataUtil.createKey(bucket, key, replicationInput.getReplicationConfig(), "Hello".getBytes(UTF_8)); + DataTestUtil.createKey(bucket, key, replicationInput.getReplicationConfig(), "Hello".getBytes(UTF_8)); List<OmKeyLocationInfo> keyLocations = lookupKey(cluster, key); assertThat(keyLocations).isNotEmpty(); OmKeyLocationInfo keyLocation = keyLocations.get(0);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/volume/TestDatanodeHddsVolumeFailureDetection.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/volume/TestDatanodeHddsVolumeFailureDetection.java index bd167bb..5c5baec 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/volume/TestDatanodeHddsVolumeFailureDetection.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/volume/TestDatanodeHddsVolumeFailureDetection.java
@@ -18,14 +18,20 @@ package org.apache.hadoop.ozone.dn.volume; import static org.apache.commons.io.IOUtils.readFully; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_NODE_REPORT_INTERVAL; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CONTAINER_CACHE_SIZE; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assumptions.assumeTrue; import java.io.File; import java.io.IOException; @@ -34,21 +40,21 @@ import java.nio.file.Paths; import java.time.Duration; import java.util.UUID; +import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.RandomUtils; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient; import org.apache.hadoop.hdds.scm.client.ScmClient; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; +import org.apache.hadoop.hdds.scm.node.DatanodeInfo; import org.apache.hadoop.hdfs.server.datanode.checker.VolumeCheckResult; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneKeyDetails; @@ -61,175 +67,219 @@ import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet; import org.apache.hadoop.ozone.container.common.volume.StorageVolume; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; +import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils; import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer; import org.apache.hadoop.ozone.dn.DatanodeTestUtils; -import org.junit.jupiter.params.ParameterizedTest; +import org.apache.ozone.test.GenericTestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.junit.jupiter.params.AfterParameterizedClassInvocation; +import org.junit.jupiter.params.BeforeParameterizedClassInvocation; +import org.junit.jupiter.params.Parameter; +import org.junit.jupiter.params.ParameterizedClass; import org.junit.jupiter.params.provider.ValueSource; /** * This class tests datanode can detect failed volumes. */ +@ParameterizedClass +@ValueSource(booleans = {true, false}) +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@Execution(ExecutionMode.SAME_THREAD) class TestDatanodeHddsVolumeFailureDetection { private static final int KEY_SIZE = 128; - @ParameterizedTest - @ValueSource(booleans = {true, false}) - void corruptChunkFile(boolean schemaV3) throws Exception { - try (MiniOzoneCluster cluster = newCluster(schemaV3)) { - try (OzoneClient client = cluster.newClient()) { - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client); + @Parameter + private boolean schemaV3; - // write a file - String keyName = UUID.randomUUID().toString(); - long containerId = createKey(bucket, keyName); + private MiniOzoneCluster cluster; + private HddsDatanodeService currentDatanode; + private long currentContainerId; - // corrupt chunk file by rename file->dir - HddsDatanodeService dn = cluster.getHddsDatanodes().get(0); - OzoneContainer oc = dn.getDatanodeStateMachine().getContainer(); - MutableVolumeSet volSet = oc.getVolumeSet(); - StorageVolume vol0 = volSet.getVolumesList().get(0); - HddsVolume volume = assertInstanceOf(HddsVolume.class, vol0); - Path chunksPath = Paths.get( - volume.getStorageDir().getPath(), - volume.getClusterID(), - Storage.STORAGE_DIR_CURRENT, - Storage.CONTAINER_DIR + "0", - String.valueOf(containerId), - OzoneConsts.STORAGE_DIR_CHUNKS - ); - File[] chunkFiles = chunksPath.toFile().listFiles(); - assertNotNull(chunkFiles); + @BeforeParameterizedClassInvocation + void initCluster() throws Exception { + cluster = newCluster(schemaV3); + } - try { - for (File chunkFile : chunkFiles) { - DatanodeTestUtils.injectDataFileFailure(chunkFile); - } + @AfterEach + void failCurrentVolume() throws Exception { + HddsDatanodeService datanode = currentDatanode; + currentDatanode = null; + if (datanode == null) { + return; + } + cluster.getStorageContainerLocationClient().closeContainer(currentContainerId); + OzoneContainer container = datanode.getDatanodeStateMachine().getContainer(); + MutableVolumeSet volumeSet = container.getVolumeSet(); + if (!volumeSet.getVolumesList().isEmpty()) { + StorageVolume volume = volumeSet.getVolumesList().get(0); + volumeSet.failVolume(volume.getStorageDir().getPath()); + container.handleVolumeFailures(); + } + waitForHandleFailedVolume(volumeSet); + GenericTestUtils.waitFor(() -> isFailedVolumeReported(datanode), 100, 10000); + } - // simulate bad volume by removing write permission on root dir - // refer to HddsVolume.check() - DatanodeTestUtils.simulateBadVolume(vol0); - - // read written file to trigger checkVolumeAsync - readKeyToTriggerCheckVolumeAsync(bucket, keyName); - - // should trigger checkVolumeAsync and - // a failed volume should be detected - DatanodeTestUtils.waitForHandleFailedVolume(volSet, 1); - } finally { - // restore for cleanup - DatanodeTestUtils.restoreBadVolume(vol0); - for (File chunkFile : chunkFiles) { - DatanodeTestUtils.restoreDataFileFromFailure(chunkFile); - } - } - } + @AfterParameterizedClassInvocation + void shutdown() { + if (cluster != null) { + cluster.close(); } } - @ParameterizedTest - @ValueSource(booleans = {true, false}) - void corruptContainerFile(boolean schemaV3) throws Exception { - try (MiniOzoneCluster cluster = newCluster(schemaV3)) { - // create a container - ContainerWithPipeline container; - OzoneConfiguration conf = cluster.getConf(); - try (ScmClient scmClient = new ContainerOperationClient(conf)) { - container = scmClient.createContainer(ReplicationType.STAND_ALONE, - ReplicationFactor.ONE, OzoneConsts.OZONE); - } + @Test + void corruptChunkFile() throws Exception { + try (OzoneClient client = cluster.newClient()) { + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client); - // corrupt container file by removing write permission on - // container metadata dir, since container update operation - // use a create temp & rename way, so we can't just rename - // container file to simulate corruption - HddsDatanodeService dn = cluster.getHddsDatanodes().get(0); + // write a file + String keyName = UUID.randomUUID().toString(); + long containerId = createKey(bucket, keyName); + currentContainerId = containerId; + + // corrupt chunk file by rename file->dir + HddsDatanodeService dn = getDatanode(containerId); + currentDatanode = dn; OzoneContainer oc = dn.getDatanodeStateMachine().getContainer(); MutableVolumeSet volSet = oc.getVolumeSet(); StorageVolume vol0 = volSet.getVolumesList().get(0); - Container<?> c1 = oc.getContainerSet().getContainer( - container.getContainerInfo().getContainerID()); - File metadataDir = new File(c1.getContainerFile().getParent()); + HddsVolume volume = assertInstanceOf(HddsVolume.class, vol0); + Path chunksPath = Paths.get( + volume.getStorageDir().getPath(), + volume.getClusterID(), + Storage.STORAGE_DIR_CURRENT, + Storage.CONTAINER_DIR + "0", + String.valueOf(containerId), + OzoneConsts.STORAGE_DIR_CHUNKS + ); + File[] chunkFiles = chunksPath.toFile().listFiles(); + assertNotNull(chunkFiles); + try { - DatanodeTestUtils.injectContainerMetaDirFailure(metadataDir); + for (File chunkFile : chunkFiles) { + DatanodeTestUtils.injectDataFileFailure(chunkFile); + } // simulate bad volume by removing write permission on root dir // refer to HddsVolume.check() DatanodeTestUtils.simulateBadVolume(vol0); - // close container to trigger checkVolumeAsync - assertThrows(IOException.class, c1::close); + // read written file to trigger checkVolumeAsync + readKeyToTriggerCheckVolumeAsync(bucket, keyName); - // should trigger CheckVolumeAsync and + // should trigger checkVolumeAsync and // a failed volume should be detected - DatanodeTestUtils.waitForHandleFailedVolume(volSet, 1); + waitForHandleFailedVolume(volSet); } finally { // restore for cleanup DatanodeTestUtils.restoreBadVolume(vol0); - DatanodeTestUtils.restoreContainerMetaDirFromFailure(metadataDir); + for (File chunkFile : chunkFiles) { + DatanodeTestUtils.restoreDataFileFromFailure(chunkFile); + } } } } - @ParameterizedTest - @ValueSource(booleans = {true, false}) - void corruptDbFile(boolean schemaV3) throws Exception { - try (MiniOzoneCluster cluster = newCluster(schemaV3)) { - try (OzoneClient client = cluster.newClient()) { - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client); + @Test + void corruptContainerFile() throws Exception { + // create a container + ContainerWithPipeline container; + OzoneConfiguration conf = cluster.getConf(); + try (ScmClient scmClient = new ContainerOperationClient(conf)) { + container = scmClient.createContainer( + RatisReplicationConfig.getInstance(ReplicationFactor.ONE), + OzoneConsts.OZONE); + } + currentContainerId = container.getContainerInfo().getContainerID(); - // write a file, will create container1 - String keyName = UUID.randomUUID().toString(); - long containerId = createKey(bucket, keyName); + // corrupt container file by removing write permission on + // container metadata dir, since container update operation + // use a create temp & rename way, so we can't just rename + // container file to simulate corruption + HddsDatanodeService dn = cluster.getHddsDatanode(container.getPipeline().getFirstNode()); + currentDatanode = dn; + OzoneContainer oc = dn.getDatanodeStateMachine().getContainer(); + MutableVolumeSet volSet = oc.getVolumeSet(); + StorageVolume vol0 = volSet.getVolumesList().get(0); + Container<?> c1 = oc.getContainerSet().getContainer( + container.getContainerInfo().getContainerID()); + File metadataDir = new File(c1.getContainerFile().getParent()); + try { + DatanodeTestUtils.injectContainerMetaDirFailure(metadataDir); - // close container1 - HddsDatanodeService dn = cluster.getHddsDatanodes().get(0); - OzoneContainer oc = dn.getDatanodeStateMachine().getContainer(); - Container<?> c1 = oc.getContainerSet().getContainer(containerId); - c1.close(); + // simulate bad volume by removing write permission on root dir + // refer to HddsVolume.check() + DatanodeTestUtils.simulateBadVolume(vol0); - // create container2, and container1 is kicked out of cache - OzoneConfiguration conf = cluster.getConf(); - try (ScmClient scmClient = new ContainerOperationClient(conf)) { - ContainerWithPipeline c2 = scmClient.createContainer( - ReplicationType.STAND_ALONE, ReplicationFactor.ONE, - OzoneConsts.OZONE); - assertEquals(c2.getContainerInfo().getState(), LifeCycleState.OPEN); - } + // close container to trigger checkVolumeAsync + assertThrows(IOException.class, c1::close); - // corrupt db by rename dir->file - File dbDir; + // should trigger CheckVolumeAsync and + // a failed volume should be detected + waitForHandleFailedVolume(volSet); + } finally { + // restore for cleanup + DatanodeTestUtils.restoreBadVolume(vol0); + DatanodeTestUtils.restoreContainerMetaDirFromFailure(metadataDir); + } + } + + @Test + void corruptDbFile() throws Exception { + try (OzoneClient client = cluster.newClient()) { + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client); + + // write a file, will create container1 + String keyName = UUID.randomUUID().toString(); + long containerId = createKey(bucket, keyName); + currentContainerId = containerId; + + // close container1 + HddsDatanodeService dn = getDatanode(containerId); + currentDatanode = dn; + OzoneContainer oc = dn.getDatanodeStateMachine().getContainer(); + Container<?> c1 = oc.getContainerSet().getContainer(containerId); + c1.close(); + + // corrupt db by rename dir->file + File dbDir; + if (schemaV3) { + dbDir = new File(((KeyValueContainerData) (c1.getContainerData())) + .getDbFile().getAbsolutePath()); + } else { + File metadataDir = new File(c1.getContainerFile().getParent()); + dbDir = new File(metadataDir, containerId + OzoneConsts.DN_CONTAINER_DB); + } + + MutableVolumeSet volSet = oc.getVolumeSet(); + StorageVolume vol0 = volSet.getVolumesList().get(0); + + try { + // remove RocksDB from cache + KeyValueContainerData containerData = (KeyValueContainerData) c1.getContainerData(); if (schemaV3) { - dbDir = new File(((KeyValueContainerData) (c1.getContainerData())) - .getDbFile().getAbsolutePath()); + DatanodeStoreCache.getInstance().removeDB(dbDir.getAbsolutePath()); } else { - File metadataDir = new File(c1.getContainerFile().getParent()); - dbDir = new File(metadataDir, "1" + OzoneConsts.DN_CONTAINER_DB); + BlockUtils.removeDB(containerData, cluster.getConf()); } + DatanodeTestUtils.injectDataDirFailure(dbDir); - MutableVolumeSet volSet = oc.getVolumeSet(); - StorageVolume vol0 = volSet.getVolumesList().get(0); + // simulate bad volume by removing write permission on root dir + // refer to HddsVolume.check() + DatanodeTestUtils.simulateBadVolume(vol0); - try { - DatanodeTestUtils.injectDataDirFailure(dbDir); - if (schemaV3) { - // remove rocksDB from cache - DatanodeStoreCache.getInstance().removeDB(dbDir.getAbsolutePath()); - } + readKeyToTriggerCheckVolumeAsync(bucket, keyName); - // simulate bad volume by removing write permission on root dir - // refer to HddsVolume.check() - DatanodeTestUtils.simulateBadVolume(vol0); - - readKeyToTriggerCheckVolumeAsync(bucket, keyName); - - // should trigger CheckVolumeAsync and - // a failed volume should be detected - DatanodeTestUtils.waitForHandleFailedVolume(volSet, 1); - } finally { - // restore all - DatanodeTestUtils.restoreBadVolume(vol0); - DatanodeTestUtils.restoreDataDirFromFailure(dbDir); - } + // should trigger CheckVolumeAsync and + // a failed volume should be detected + waitForHandleFailedVolume(volSet); + } finally { + // restore all + DatanodeTestUtils.restoreBadVolume(vol0); + DatanodeTestUtils.restoreDataDirFromFailure(dbDir); } } } @@ -239,65 +289,47 @@ void corruptDbFile(boolean schemaV3) throws Exception { * test to reach the helper method {@link HddsVolume#checkDbHealth}. * As a workaround, we test the helper method directly. * As we test the helper method directly, we cannot test for schemas older than V3. - * - * @param schemaV3 - * @throws Exception */ - @ParameterizedTest - @ValueSource(booleans = {true}) - void corruptDbFileWithoutDbHandleCacheInvalidation(boolean schemaV3) throws Exception { - try (MiniOzoneCluster cluster = newCluster(schemaV3)) { - try (OzoneClient client = cluster.newClient()) { - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client); + @Test + void corruptDbFileWithoutDbHandleCacheInvalidation() throws Exception { + assumeTrue(schemaV3); + try (OzoneClient client = cluster.newClient()) { + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client); - // write a file, will create container1 - String keyName = UUID.randomUUID().toString(); - long containerId = createKey(bucket, keyName); + // write a file, will create container1 + String keyName = UUID.randomUUID().toString(); + long containerId = createKey(bucket, keyName); + currentContainerId = containerId; - // close container1 - HddsDatanodeService dn = cluster.getHddsDatanodes().get(0); - OzoneContainer oc = dn.getDatanodeStateMachine().getContainer(); - Container<?> c1 = oc.getContainerSet().getContainer(containerId); - c1.close(); + // close container1 + HddsDatanodeService dn = getDatanode(containerId); + currentDatanode = dn; + OzoneContainer oc = dn.getDatanodeStateMachine().getContainer(); + Container<?> c1 = oc.getContainerSet().getContainer(containerId); + c1.close(); - // create container2, and container1 is kicked out of cache - OzoneConfiguration conf = cluster.getConf(); - try (ScmClient scmClient = new ContainerOperationClient(conf)) { - ContainerWithPipeline c2 = scmClient.createContainer( - ReplicationType.STAND_ALONE, ReplicationFactor.ONE, - OzoneConsts.OZONE); - assertEquals(c2.getContainerInfo().getState(), LifeCycleState.OPEN); - } + // corrupt db by rename dir->file + File dbDir = new File(((KeyValueContainerData) (c1.getContainerData())) + .getDbFile().getAbsolutePath()); - // corrupt db by rename dir->file - File dbDir; - if (schemaV3) { - dbDir = new File(((KeyValueContainerData) (c1.getContainerData())) - .getDbFile().getAbsolutePath()); - } else { - File metadataDir = new File(c1.getContainerFile().getParent()); - dbDir = new File(metadataDir, "1" + OzoneConsts.DN_CONTAINER_DB); - } + MutableVolumeSet volSet = oc.getVolumeSet(); + HddsVolume vol0 = (HddsVolume) volSet.getVolumesList().get(0); - MutableVolumeSet volSet = oc.getVolumeSet(); - HddsVolume vol0 = (HddsVolume) volSet.getVolumesList().get(0); + try { + DatanodeTestUtils.injectDataDirFailure(dbDir); + // simulate bad volume by removing write permission on root dir + // refer to HddsVolume.check() + DatanodeTestUtils.simulateBadVolume(vol0); - try { - DatanodeTestUtils.injectDataDirFailure(dbDir); - // simulate bad volume by removing write permission on root dir - // refer to HddsVolume.check() - DatanodeTestUtils.simulateBadVolume(vol0); - - // one volume health check got automatically executed when the cluster started - // the second health should log the rocksdb failure but return a healthy-volume status - assertEquals(VolumeCheckResult.HEALTHY, vol0.checkDbHealth(dbDir)); - // the third health check should log the rocksdb failure and return a failed-volume status - assertEquals(VolumeCheckResult.FAILED, vol0.checkDbHealth(dbDir)); - } finally { - // restore all - DatanodeTestUtils.restoreBadVolume(vol0); - DatanodeTestUtils.restoreDataDirFromFailure(dbDir); - } + // one volume health check got automatically executed when the cluster started + // the second health should log the rocksdb failure but return a healthy-volume status + assertEquals(VolumeCheckResult.HEALTHY, vol0.checkDbHealth(dbDir)); + // the third health check should log the rocksdb failure and return a failed-volume status + assertEquals(VolumeCheckResult.FAILED, vol0.checkDbHealth(dbDir)); + } finally { + // restore all + DatanodeTestUtils.restoreBadVolume(vol0); + DatanodeTestUtils.restoreDataDirFromFailure(dbDir); } } } @@ -309,6 +341,24 @@ private static void readKeyToTriggerCheckVolumeAsync(OzoneBucket bucket, } } + private HddsDatanodeService getDatanode(long containerId) throws IOException { + try (ScmClient scmClient = new ContainerOperationClient(cluster.getConf())) { + return cluster.getHddsDatanode(scmClient.getContainerWithPipeline(containerId) + .getPipeline().getFirstNode()); + } + } + + private boolean isFailedVolumeReported(HddsDatanodeService datanode) { + DatanodeInfo datanodeInfo = cluster.getStorageContainerManager().getScmNodeManager() + .getNode(datanode.getDatanodeDetails().getID()); + return datanodeInfo != null && datanodeInfo.getFailedVolumeCount() == 1; + } + + private static void waitForHandleFailedVolume(MutableVolumeSet volumeSet) throws Exception { + DatanodeTestUtils.waitForHandleFailedVolume(volumeSet, 1); + GenericTestUtils.waitFor(() -> volumeSet.getVolumesList().isEmpty(), 100, 10000); + } + private static MiniOzoneCluster newCluster(boolean schemaV3) throws Exception { OzoneConfiguration ozoneConfig = new OzoneConfiguration(); @@ -319,6 +369,11 @@ private static MiniOzoneCluster newCluster(boolean schemaV3) // keep the cache size = 1, so we could trigger io exception on // reading on-disk db instance ozoneConfig.setInt(OZONE_CONTAINER_CACHE_SIZE, 1); + ozoneConfig.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 100, TimeUnit.MILLISECONDS); + ozoneConfig.setTimeDuration(HDDS_NODE_REPORT_INTERVAL, 100, TimeUnit.MILLISECONDS); + ozoneConfig.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100, TimeUnit.MILLISECONDS); + ozoneConfig.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, TimeUnit.SECONDS); + ozoneConfig.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, TimeUnit.SECONDS); if (!schemaV3) { ContainerTestUtils.disableSchemaV3(ozoneConfig); } @@ -330,7 +385,7 @@ private static MiniOzoneCluster newCluster(boolean schemaV3) dnConf.setDiskCheckMinGap(Duration.ofSeconds(0)); ozoneConfig.setFromObject(dnConf); MiniOzoneCluster cluster = MiniOzoneCluster.newBuilder(ozoneConfig) - .setNumDatanodes(1) + .setNumDatanodes(4) .build(); cluster.waitForClusterToBeReady(); cluster.waitForPipelineTobeReady(ReplicationFactor.ONE, 30000); @@ -343,7 +398,7 @@ private static long createKey(OzoneBucket bucket, String key) byte[] bytes = RandomUtils.secure().randomBytes(KEY_SIZE); RatisReplicationConfig replication = RatisReplicationConfig.getInstance(ReplicationFactor.ONE); - TestDataUtil.createKey(bucket, key, replication, bytes); + DataTestUtil.createKey(bucket, key, replication, bytes); OzoneKeyDetails keyDetails = bucket.getKey(key); assertEquals(key, keyDetails.getName()); return keyDetails.getOzoneKeyLocations().get(0).getContainerID();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteFileOps.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteFileOps.java index 38c7766..2248966 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteFileOps.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteFileOps.java
@@ -27,8 +27,8 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.freon.OmBucketTestUtils.ParameterBuilder; import org.apache.ozone.test.NonHATests; @@ -106,7 +106,7 @@ static List<ParameterBuilder> parameters() { @MethodSource("parameters") void testOmBucketReadWriteFileOps(ParameterBuilder parameterBuilder) throws Exception { try (OzoneClient client = cluster().newClient()) { - TestDataUtil.createVolumeAndBucket(client, + DataTestUtil.createVolumeAndBucket(client, parameterBuilder.getVolumeName(), parameterBuilder.getBucketName(), parameterBuilder.getBucketArgs().build()
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteKeyOps.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteKeyOps.java index dc6c343..1cf361d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteKeyOps.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteKeyOps.java
@@ -26,7 +26,7 @@ import java.util.Iterator; import java.util.List; import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.ozone.TestDataUtil; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneKey; @@ -115,7 +115,7 @@ static List<ParameterBuilder> parameters() { @ParameterizedTest(name = "{0}") @MethodSource("parameters") void testOmBucketReadWriteKeyOps(ParameterBuilder parameterBuilder) throws Exception { - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client, + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client, parameterBuilder.getVolumeName(), parameterBuilder.getBucketName(), parameterBuilder.getBucketArgs().setBucketLayout(BucketLayout.OBJECT_STORE).build()
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneClusterRuntime.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneClusterRuntime.java index a4a10f0..305eabd 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneClusterRuntime.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneClusterRuntime.java
@@ -29,7 +29,7 @@ import java.time.Duration; import java.util.UUID; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.ozone.TestDataUtil; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; @@ -99,11 +99,11 @@ private void startRuntimeAndCreateKey(LocalOzoneClusterConfig config, try (OzoneClient client = OzoneClientFactory.getRpcClient(clientConf)) { OzoneBucket bucket = - TestDataUtil.createVolumeAndBucket(client, volumeName, bucketName); + DataTestUtil.createVolumeAndBucket(client, volumeName, bucketName); // Writing and reading back a key proves the datanodes registered and // SCM left safe mode, so the cluster is actually usable. - TestDataUtil.createKey(bucket, keyName, KEY_CONTENT.getBytes(UTF_8)); - assertEquals(KEY_CONTENT, TestDataUtil.getKey(bucket, keyName)); + DataTestUtil.createKey(bucket, keyName, KEY_CONTENT.getBytes(UTF_8)); + assertEquals(KEY_CONTENT, DataTestUtil.getKey(bucket, keyName)); } } } @@ -124,7 +124,7 @@ private void restartRuntimeAndVerifyKey(LocalOzoneClusterConfig config, assertEquals(bucketName, bucket.getName()); // Key data written before the restart is still readable from the // persistent datanode storage. - assertEquals(KEY_CONTENT, TestDataUtil.getKey(bucket, keyName)); + assertEquals(KEY_CONTENT, DataTestUtil.getKey(bucket, keyName)); } } }
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/BucketForTesting.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/BucketForTesting.java index 4c07c09..724a4d4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/BucketForTesting.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/BucketForTesting.java
@@ -27,7 +27,7 @@ import java.util.concurrent.ThreadLocalRandom; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; -import org.apache.hadoop.ozone.TestDataUtil; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -78,7 +78,7 @@ public KeyInputStream getKeyInputStream(String keyName) throws IOException { public void writeKey(String key, ReplicationConfig repConfig, byte[] inputData) throws IOException { - TestDataUtil.createKey(bucket, key, repConfig, inputData); + DataTestUtil.createKey(bucket, key, repConfig, inputData); } public byte[] writeRandomBytes(String keyName, int dataLength)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketLayoutWithOlderClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketLayoutWithOlderClient.java index 08961fc..71a9ab3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketLayoutWithOlderClient.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketLayoutWithOlderClient.java
@@ -24,8 +24,8 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.StorageTypeProto; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -58,17 +58,17 @@ public void testCreateBucketWithOlderClient() throws Exception { // create a volume and a bucket without bucket layout argument BucketLayout defaultLayout = cluster().getConf().getEnum(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT, BucketLayout.FILE_SYSTEM_OPTIMIZED); - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client, null); + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client, null); String volumeName = bucket.getVolumeName(); // OM defaulted bucket layout assertEquals(defaultLayout, bucket.getBucketLayout()); // Sets bucket layout explicitly. - OzoneBucket fsobucket = TestDataUtil + OzoneBucket fsobucket = DataTestUtil .createVolumeAndBucket(client, BucketLayout.FILE_SYSTEM_OPTIMIZED); assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, fsobucket.getBucketLayout()); - OzoneBucket obsBucket = TestDataUtil.createVolumeAndBucket(client, BucketLayout.OBJECT_STORE); + OzoneBucket obsBucket = DataTestUtil.createVolumeAndBucket(client, BucketLayout.OBJECT_STORE); assertEquals(BucketLayout.OBJECT_STORE, obsBucket.getBucketLayout()); // Create bucket request by an older client.
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketOwner.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketOwner.java index 918e738..642be2a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketOwner.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketOwner.java
@@ -17,8 +17,8 @@ package org.apache.hadoop.ozone.om; +import static org.apache.hadoop.ozone.DataTestUtil.createKey; import static org.apache.hadoop.ozone.OzoneAcl.AclScope.DEFAULT; -import static org.apache.hadoop.ozone.TestDataUtil.createKey; import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType.USER; import static org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType.OZONE; import static org.junit.jupiter.api.Assertions.assertThrows;
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyPurging.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyPurging.java index 2a09ffc..0f1973c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyPurging.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyPurging.java
@@ -30,8 +30,8 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -103,7 +103,7 @@ public void testKeysPurgingByKeyDeletingService() throws Exception { for (int i = 1; i <= NUM_KEYS; i++) { String keyName = keyBase + "-" + i; keys.add(keyName); - TestDataUtil.createKey(bucket, keyName, data); + DataTestUtil.createKey(bucket, keyName, data); } // Delete created keys
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeys.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeys.java index 8772216..e983c93 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeys.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeys.java
@@ -36,7 +36,7 @@ import org.apache.commons.io.IOUtils; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.ozone.TestDataUtil; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; @@ -79,11 +79,11 @@ void init() throws Exception { client = OzoneClientFactory.getRpcClient(conf); // create a volume and a LEGACY bucket - legacyOzoneBucket = TestDataUtil + legacyOzoneBucket = DataTestUtil .createVolumeAndBucket(client, BucketLayout.LEGACY); // create a volume and a OBJECT_STORE bucket - obsOzoneBucket = TestDataUtil + obsOzoneBucket = DataTestUtil .createVolumeAndBucket(client, BucketLayout.OBJECT_STORE); initFSNameSpace(); @@ -369,7 +369,7 @@ private void checkKeyShallowList(String keyPrefix, String startKey, private static void createAndAssertKeys(OzoneBucket ozoneBucket, List<String> keys) throws Exception { for (String key : keys) { - byte[] input = TestDataUtil.createStringKey(ozoneBucket, key, 10); + byte[] input = DataTestUtil.createStringKey(ozoneBucket, key, 10); // Read the key with given key name. readkey(ozoneBucket, key, 10, input); }
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java index 30d38a6..6929dac 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java
@@ -35,7 +35,7 @@ import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.StorageType; -import org.apache.hadoop.ozone.TestDataUtil; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -83,7 +83,7 @@ void init() throws Exception { client = OzoneClientFactory.getRpcClient(conf); // create a volume and a LEGACY bucket - legacyOzoneBucket = TestDataUtil + legacyOzoneBucket = DataTestUtil .createVolumeAndBucket(client, BucketLayout.LEGACY); String volumeName = legacyOzoneBucket.getVolumeName(); @@ -662,7 +662,7 @@ private void checkKeyShallowList(String keyPrefix, String startKey, private static void createAndAssertKeys(OzoneBucket ozoneBucket, List<String> keys) throws Exception { for (String key : keys) { - byte[] input = TestDataUtil.createStringKey(ozoneBucket, key, 10); + byte[] input = DataTestUtil.createStringKey(ozoneBucket, key, 10); // Read the key with given key name. readkey(ozoneBucket, key, 10, input); }
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java index 2e4f236..0406f34 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.ozone.TestDataUtil; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; @@ -65,7 +65,7 @@ void init() throws Exception { client = OzoneClientFactory.getRpcClient(conf); // create a volume and a LEGACY bucket - fsoOzoneBucket = TestDataUtil + fsoOzoneBucket = DataTestUtil .createVolumeAndBucket(client, BucketLayout.FILE_SYSTEM_OPTIMIZED); buildNameSpaceTree(fsoOzoneBucket);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index d9fe8a7..34f8a32 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java
@@ -105,9 +105,9 @@ import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; import org.apache.hadoop.hdds.utils.db.DBStore; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.lock.BootstrapStateHandler; @@ -705,14 +705,14 @@ private void setupGetMethod(Collection<String> toExcludeList) { private void prepSnapshotData() throws Exception { metaDir = OMStorage.getOmDbDir(conf); - OzoneBucket bucket = TestDataUtil + OzoneBucket bucket = DataTestUtil .createVolumeAndBucket(client); // Create dummy keys for snapshotting. - TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), ReplicationConfig + DataTestUtil.createKey(bucket, UUID.randomUUID().toString(), ReplicationConfig .fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), "content".getBytes(StandardCharsets.UTF_8)); - TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), ReplicationConfig + DataTestUtil.createKey(bucket, UUID.randomUUID().toString(), ReplicationConfig .fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), "content".getBytes(StandardCharsets.UTF_8));
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServletInodeBasedXfer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServletInodeBasedXfer.java index 518c762..8bf1fbf 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServletInodeBasedXfer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServletInodeBasedXfer.java
@@ -105,9 +105,9 @@ import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.hdds.utils.db.InodeMetadataRocksDBCheckpoint; import org.apache.hadoop.hdfs.web.URLConnectionFactory; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneSnapshot; @@ -556,9 +556,9 @@ public void testBootstrapOnFollowerConsistency() throws Exception { om.getKeyManager().getSnapshotSstFilteringService().pause(); om.getKeyManager().getSnapshotDeletingService().suspend(); // Create test data and snapshots - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client, volumeName, bucketName); + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client, volumeName, bucketName); // Create key before first snapshot - TestDataUtil.createKey(bucket, "key1", + DataTestUtil.createKey(bucket, "key1", ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), "data1".getBytes(StandardCharsets.UTF_8)); client.getObjectStore().createSnapshot(volumeName, bucketName, "snapshot1"); @@ -821,10 +821,10 @@ public void testCheckpointIncludesSnapshotsFromFrozenState() throws Exception { om.getKeyManager().getSnapshotSstFilteringService().pause(); // Create test data and snapshots - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client, volumeName, bucketName); + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client, volumeName, bucketName); // Create key before first snapshot - TestDataUtil.createKey(bucket, "key1", + DataTestUtil.createKey(bucket, "key1", ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), "data1".getBytes(StandardCharsets.UTF_8)); client.getObjectStore().createSnapshot(volumeName, bucketName, "snapshot1"); @@ -1134,18 +1134,18 @@ private static String getInode(String inodeAndMtime) { } private void writeData(String volumeName, String bucketName, boolean includeSnapshots) throws Exception { - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client, volumeName, bucketName); + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client, volumeName, bucketName); for (int i = 0; i < 10; i++) { - TestDataUtil.createKey(bucket, "key" + i, + DataTestUtil.createKey(bucket, "key" + i, ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), "sample".getBytes(StandardCharsets.UTF_8)); om.getMetadataManager().getStore().flushDB(); } if (includeSnapshots) { - TestDataUtil.createKey(bucket, "keysnap1", + DataTestUtil.createKey(bucket, "keysnap1", ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), "sample".getBytes(StandardCharsets.UTF_8)); - TestDataUtil.createKey(bucket, "keysnap2", + DataTestUtil.createKey(bucket, "keysnap2", ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), "sample".getBytes(StandardCharsets.UTF_8)); client.getObjectStore().createSnapshot(volumeName, bucketName, "snapshot10");
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshotTransfer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshotTransfer.java index ba5d340..12551fd 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshotTransfer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshotTransfer.java
@@ -17,8 +17,8 @@ package org.apache.hadoop.ozone.om; +import static org.apache.hadoop.ozone.DataTestUtil.readFully; import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; -import static org.apache.hadoop.ozone.TestDataUtil.readFully; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_MAX_TOTAL_SST_SIZE_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.om.OmSnapshotManager.OM_HARDLINK_FILE;
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index cb9d0d8..1925b6a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java
@@ -18,8 +18,8 @@ package org.apache.hadoop.ozone.om; import static org.apache.hadoop.hdds.utils.IOUtils.getINode; +import static org.apache.hadoop.ozone.DataTestUtil.readFully; import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; -import static org.apache.hadoop.ozone.TestDataUtil.readFully; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath; import static org.apache.hadoop.ozone.om.TestOzoneManagerHAWithStoppedNodes.createKey; import static org.apache.ozone.test.OzoneTestBase.uniqueObjectName;
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMStartupWithBucketLayout.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMStartupWithBucketLayout.java index aec305c..2d2e8ef 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMStartupWithBucketLayout.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMStartupWithBucketLayout.java
@@ -22,8 +22,8 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -69,19 +69,19 @@ public void testRestartWithFSOLayout() throws Exception { startCluster(conf); // 2. create bucket with FSO bucket layout and verify - OzoneBucket bucket1 = TestDataUtil.createVolumeAndBucket(client, + OzoneBucket bucket1 = DataTestUtil.createVolumeAndBucket(client, BucketLayout.FILE_SYSTEM_OPTIMIZED); verifyBucketLayout(bucket1, BucketLayout.FILE_SYSTEM_OPTIMIZED); // 3. verify OM default behavior with empty restartCluster(); - OzoneBucket bucket2 = TestDataUtil.createVolumeAndBucket(client, + OzoneBucket bucket2 = DataTestUtil.createVolumeAndBucket(client, null); verifyBucketLayout(bucket2, BucketLayout.FILE_SYSTEM_OPTIMIZED); // 4. create bucket with OBS bucket layout and verify restartCluster(); - OzoneBucket bucket3 = TestDataUtil.createVolumeAndBucket(client, + OzoneBucket bucket3 = DataTestUtil.createVolumeAndBucket(client, BucketLayout.OBJECT_STORE); verifyBucketLayout(bucket3, BucketLayout.OBJECT_STORE); @@ -113,19 +113,19 @@ public void testRestartWithOBSLayout() throws Exception { startCluster(conf); // 2. create bucket with FSO bucket layout and verify - OzoneBucket bucket1 = TestDataUtil.createVolumeAndBucket(client, + OzoneBucket bucket1 = DataTestUtil.createVolumeAndBucket(client, BucketLayout.FILE_SYSTEM_OPTIMIZED); verifyBucketLayout(bucket1, BucketLayout.FILE_SYSTEM_OPTIMIZED); // 3. verify OM default behavior with empty restartCluster(); - OzoneBucket bucket2 = TestDataUtil.createVolumeAndBucket(client, + OzoneBucket bucket2 = DataTestUtil.createVolumeAndBucket(client, null); verifyBucketLayout(bucket2, BucketLayout.OBJECT_STORE); // 4. create bucket with OBS bucket layout and verify restartCluster(); - OzoneBucket bucket3 = TestDataUtil.createVolumeAndBucket(client, + OzoneBucket bucket3 = DataTestUtil.createVolumeAndBucket(client, BucketLayout.OBJECT_STORE); verifyBucketLayout(bucket3, BucketLayout.OBJECT_STORE);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java index 0802312..025a472 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java
@@ -54,10 +54,10 @@ import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; @@ -100,7 +100,7 @@ void init() throws Exception { cluster = cluster(); client = cluster.newClient(); // create a volume and a bucket to be used by OzoneFileSystem - OzoneBucket bucket = TestDataUtil + OzoneBucket bucket = DataTestUtil .createVolumeAndBucket(client, BucketLayout.FILE_SYSTEM_OPTIMIZED); volumeName = bucket.getVolumeName(); bucketName = bucket.getName(); @@ -210,7 +210,7 @@ public void testCreateKey() throws Exception { @Test public void testDeleteBucketWithKeys() throws Exception { // Create temporary volume and bucket for this test. - OzoneBucket testBucket = TestDataUtil + OzoneBucket testBucket = DataTestUtil .createVolumeAndBucket(client, BucketLayout.FILE_SYSTEM_OPTIMIZED); String testVolumeName = testBucket.getVolumeName(); String testBucketName = testBucket.getName(); @@ -556,7 +556,7 @@ private void createAndAssertKeys(OzoneBucket ozoneBucket, List<String> keys) throws Exception { for (String key : keys) { - byte[] input = TestDataUtil.createStringKey(ozoneBucket, key, 10); + byte[] input = DataTestUtil.createStringKey(ozoneBucket, key, 10); // Read the key with given key name. readKey(ozoneBucket, key, 10, input); }
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithLegacyFS.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithLegacyFS.java index 71a3ac2..7904666 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithLegacyFS.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithLegacyFS.java
@@ -37,8 +37,8 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -100,7 +100,7 @@ public void init() throws Exception { bucketName = RandomStringUtils.secure().nextAlphabetic(10).toLowerCase(); // create a volume and a bucket to be used by OzoneFileSystem - TestDataUtil.createVolumeAndBucket(client, volumeName, bucketName, + DataTestUtil.createVolumeAndBucket(client, volumeName, bucketName, BucketLayout.OBJECT_STORE); volume = client.getObjectStore().getVolume(volumeName); }
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java index 8faf7d9..90d6b12 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java
@@ -35,9 +35,9 @@ import java.util.Collections; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneAcl; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.audit.AuditEventStatus; import org.apache.hadoop.ozone.audit.AuditLogTestUtils; import org.apache.hadoop.ozone.audit.OMAction; @@ -112,7 +112,7 @@ public void testCreateVolumePermissionDenied() throws Exception { authorizer.volumeAclAllow = false; OMException exception = assertThrows(OMException.class, - () -> TestDataUtil.createVolumeAndBucket(client)); + () -> DataTestUtil.createVolumeAndBucket(client)); assertEquals(ResultCodes.PERMISSION_DENIED, exception.getResult()); assertThat(logCapturer.getOutput()) @@ -122,7 +122,7 @@ public void testCreateVolumePermissionDenied() throws Exception { @Test public void testReadVolumePermissionDenied() throws Exception { - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client); + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client); authorizer.volumeAclAllow = false; ObjectStore objectStore = client.getObjectStore(); OMException exception = assertThrows(OMException.class, () -> @@ -139,7 +139,7 @@ public void testCreateBucketPermissionDenied() throws Exception { authorizer.bucketAclAllow = false; OMException exception = assertThrows(OMException.class, - () -> TestDataUtil.createVolumeAndBucket(client)); + () -> DataTestUtil.createVolumeAndBucket(client)); assertEquals(ResultCodes.PERMISSION_DENIED, exception.getResult()); assertThat(logCapturer.getOutput()) @@ -149,7 +149,7 @@ public void testCreateBucketPermissionDenied() throws Exception { @Test public void testReadBucketPermissionDenied() throws Exception { - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client); + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client); authorizer.bucketAclAllow = false; ObjectStore objectStore = client.getObjectStore(); OMException exception = assertThrows(OMException.class, @@ -167,10 +167,10 @@ public void testReadBucketPermissionDenied() throws Exception { public void testCreateKeyPermissionDenied() throws Exception { authorizer.keyAclAllow = false; - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client); + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client); OMException exception = assertThrows(OMException.class, - () -> TestDataUtil.createKey(bucket, "testKey", "testcontent".getBytes(StandardCharsets.UTF_8))); + () -> DataTestUtil.createKey(bucket, "testKey", "testcontent".getBytes(StandardCharsets.UTF_8))); assertEquals(ResultCodes.PERMISSION_DENIED, exception.getResult()); assertThat(logCapturer.getOutput()).contains("doesn't have CREATE " + "permission to access key"); @@ -178,12 +178,12 @@ public void testCreateKeyPermissionDenied() throws Exception { @Test public void testReadKeyPermissionDenied() throws Exception { - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client); - TestDataUtil.createKey(bucket, "testKey", "testcontent".getBytes(StandardCharsets.UTF_8)); + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client); + DataTestUtil.createKey(bucket, "testKey", "testcontent".getBytes(StandardCharsets.UTF_8)); authorizer.keyAclAllow = false; OMException exception = assertThrows(OMException.class, - () -> TestDataUtil.getKey(bucket, "testKey")); + () -> DataTestUtil.getKey(bucket, "testKey")); assertEquals(ResultCodes.PERMISSION_DENIED, exception.getResult()); assertThat(logCapturer.getOutput()).contains("doesn't have READ " + @@ -193,8 +193,8 @@ public void testReadKeyPermissionDenied() throws Exception { @Test public void testGetFileStatusPermissionDenied() throws Exception { - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client); - TestDataUtil.createKey(bucket, "testKey", "testcontent".getBytes(StandardCharsets.UTF_8)); + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client); + DataTestUtil.createKey(bucket, "testKey", "testcontent".getBytes(StandardCharsets.UTF_8)); authorizer.keyAclAllow = false; OMException exception = assertThrows(OMException.class, @@ -208,7 +208,7 @@ public void testGetFileStatusPermissionDenied() throws Exception { @Test public void testSetACLPermissionDenied() throws Exception { - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client); + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client); authorizer.bucketAclAllow = false; @@ -222,9 +222,9 @@ public void testSetACLPermissionDenied() throws Exception { @Test public void testKeyACLOpsPermissionDenied() throws Exception { - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client); + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client); String keyName = "testKey"; - TestDataUtil.createKey(bucket, keyName, "testcontent".getBytes(StandardCharsets.UTF_8)); + DataTestUtil.createKey(bucket, keyName, "testcontent".getBytes(StandardCharsets.UTF_8)); authorizer.keyAclAllow = false; ObjectStore objectStore = client.getObjectStore();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmBlockVersioning.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmBlockVersioning.java index 85ba7c9..4632277 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmBlockVersioning.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmBlockVersioning.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.ozone.TestDataUtil; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; @@ -73,7 +73,7 @@ public void testAllocateCommit() throws Exception { String keyName = uniqueObjectName("key"); OzoneBucket bucket = - TestDataUtil.createVolumeAndBucket(client, volumeName, bucketName); + DataTestUtil.createVolumeAndBucket(client, volumeName, bucketName); // Versioning isn't supported currently, but just preserving old behaviour bucket.setVersioning(true); @@ -155,7 +155,7 @@ public void testReadLatestVersion() throws Exception { String keyName = uniqueObjectName("key"); OzoneBucket bucket = - TestDataUtil.createVolumeAndBucket(client, volumeName, bucketName); + DataTestUtil.createVolumeAndBucket(client, volumeName, bucketName); OmKeyArgs omKeyArgs = new OmKeyArgs.Builder() .setVolumeName(volumeName) @@ -166,8 +166,8 @@ public void testReadLatestVersion() throws Exception { String dataString = RandomStringUtils.secure().nextAlphabetic(100); - TestDataUtil.createKey(bucket, keyName, dataString.getBytes(StandardCharsets.UTF_8)); - assertEquals(dataString, TestDataUtil.getKey(bucket, keyName)); + DataTestUtil.createKey(bucket, keyName, dataString.getBytes(StandardCharsets.UTF_8)); + assertEquals(dataString, DataTestUtil.getKey(bucket, keyName)); OmKeyInfo keyInfo = ozoneManager.lookupKey(omKeyArgs); assertEquals(0, keyInfo.getLatestVersionLocations().getVersion()); assertEquals(1, @@ -175,19 +175,19 @@ public void testReadLatestVersion() throws Exception { // When bucket versioning is disabled, overwriting a key doesn't increment // its version count. Rather it always resets the version to 0 - TestDataUtil.createKey(bucket, keyName, dataString.getBytes(StandardCharsets.UTF_8)); + DataTestUtil.createKey(bucket, keyName, dataString.getBytes(StandardCharsets.UTF_8)); keyInfo = ozoneManager.lookupKey(omKeyArgs); - assertEquals(dataString, TestDataUtil.getKey(bucket, keyName)); + assertEquals(dataString, DataTestUtil.getKey(bucket, keyName)); assertEquals(0, keyInfo.getLatestVersionLocations().getVersion()); assertEquals(1, keyInfo.getLatestVersionLocations().getLocationList().size()); dataString = RandomStringUtils.secure().nextAlphabetic(200); - TestDataUtil.createKey(bucket, keyName, dataString.getBytes(StandardCharsets.UTF_8)); + DataTestUtil.createKey(bucket, keyName, dataString.getBytes(StandardCharsets.UTF_8)); keyInfo = ozoneManager.lookupKey(omKeyArgs); - assertEquals(dataString, TestDataUtil.getKey(bucket, keyName)); + assertEquals(dataString, DataTestUtil.getKey(bucket, keyName)); assertEquals(0, keyInfo.getLatestVersionLocations().getVersion()); assertEquals(1, keyInfo.getLatestVersionLocations().getLocationList().size());
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java index add7f83..92974a6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
@@ -61,10 +61,10 @@ import org.apache.hadoop.hdds.utils.db.RocksDatabaseException; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -352,7 +352,7 @@ public void testKeyOps() throws Exception { long initialNumDeleteObjectTaggingFails = getLongCounter("NumDeleteObjectTaggingFails", omMetrics); // see HDDS-10078 for making this work with FILE_SYSTEM_OPTIMIZED layout - TestDataUtil.createVolumeAndBucket(client, volumeName, bucketName, BucketLayout.LEGACY); + DataTestUtil.createVolumeAndBucket(client, volumeName, bucketName, BucketLayout.LEGACY); OmKeyArgs keyArgs = createKeyArgs(volumeName, bucketName, RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE)); doKeyOps(keyArgs); // This will perform 7 different operations on the key @@ -483,7 +483,7 @@ public void testDirectoryOps(BucketLayout bucketLayout) throws Exception { String bucketName = UUID.randomUUID().toString(); // create bucket with different layout in each ParameterizedTest - TestDataUtil.createVolumeAndBucket(client, volumeName, bucketName, bucketLayout); + DataTestUtil.createVolumeAndBucket(client, volumeName, bucketName, bucketLayout); // Create bucket with 2 nested directories. String rootPath = String.format("%s://%s/",
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerPrepare.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerPrepare.java index 93eeae9..3183d6e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerPrepare.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerPrepare.java
@@ -39,8 +39,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.protocol.ClientProtocol; @@ -388,7 +388,7 @@ private void writeTestData(String volumeName, String keyString = UUID.randomUUID().toString(); byte[] data = ContainerTestHelper.getFixedLengthString( keyString, 100).getBytes(UTF_8); - TestDataUtil.createKey(store.getVolume(volumeName). + DataTestUtil.createKey(store.getVolume(volumeName). getBucket(bucketName), keyName, data); }
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestRecursiveAclWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestRecursiveAclWithFSO.java index 70f2ff5..9df0d34 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestRecursiveAclWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestRecursiveAclWithFSO.java
@@ -17,7 +17,7 @@ package org.apache.hadoop.ozone.om; -import static org.apache.hadoop.ozone.TestDataUtil.createKey; +import static org.apache.hadoop.ozone.DataTestUtil.createKey; import static org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType.OZONE; import static org.apache.ozone.test.OzoneTestBase.uniqueObjectName; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -31,8 +31,8 @@ import java.util.List; import java.util.UUID; import org.apache.hadoop.hdds.protocol.StorageType; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.OzoneAcl; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; @@ -330,7 +330,7 @@ private void createKeys(ObjectStore objectStore, OzoneBucket ozoneBucket, String aclWorldAll = "world::a"; for (String key : keys) { - TestDataUtil.createStringKey(ozoneBucket, key, 10); + DataTestUtil.createStringKey(ozoneBucket, key, 10); setKeyAcl(objectStore, ozoneBucket.getVolumeName(), ozoneBucket.getName(), key, aclWorldAll); }
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmSafeMode.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmSafeMode.java index fcb7478..aba4812 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmSafeMode.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmSafeMode.java
@@ -60,9 +60,9 @@ import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.server.events.EventQueue; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -124,7 +124,7 @@ public void shutdown() { @Test void testSafeModeOperations() throws Exception { - TestDataUtil.createKeys(cluster, 100); + DataTestUtil.createKeys(cluster, 100); final List<ContainerInfo> containers = cluster .getStorageContainerManager().getContainerManager().getContainers(); GenericTestUtils.waitFor(() -> containers.size() >= 3, 100, 1000); @@ -204,7 +204,7 @@ void testClusterExitsSafeModeWithPeriodicRuleRefresh() throws Exception { cluster = builder.build(); cluster.waitForClusterToBeReady(); final StorageContainerManager scm = cluster.getStorageContainerManager(); - TestDataUtil.createKeys(cluster, 100); + DataTestUtil.createKeys(cluster, 100); GenericTestUtils.waitFor(() -> scm.getContainerManager().getContainers().size() >= 3, 100, 1000 * 30); @@ -233,7 +233,7 @@ void testSCMSafeMode() throws Exception { assertFalse(cluster.getStorageContainerManager().isInSafeMode()); // Test2: Test safe mode when containers are there in system. - TestDataUtil.createKeys(cluster, 100 * 2); + DataTestUtil.createKeys(cluster, 100 * 2); final List<ContainerInfo> containers = cluster .getStorageContainerManager().getContainerManager().getContainers(); GenericTestUtils.waitFor(() -> containers.size() >= 3, 100, 1000 * 30); @@ -304,7 +304,7 @@ public void testSCMSafeModeRestrictedOp() throws Exception { cluster.waitTobeOutOfSafeMode(); assertFalse(scm.isInSafeMode()); - TestDataUtil.createKeys(cluster, 10); + DataTestUtil.createKeys(cluster, 10); SCMClientProtocolServer clientProtocolServer = cluster .getStorageContainerManager().getClientProtocolServer(); assertFalse((scm.getClientProtocolServer()).getSafeModeStatus());
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingServiceWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingServiceWithFSO.java index e64a3f0..d516a74 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingServiceWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingServiceWithFSO.java
@@ -54,9 +54,9 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; @@ -122,7 +122,7 @@ public static void init() throws Exception { client = cluster.newClient(); // create a volume and a bucket to be used by OzoneFileSystem - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client, + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client, BucketLayout.FILE_SYSTEM_OPTIMIZED); volumeName = bucket.getVolumeName(); bucketName = bucket.getName();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestRootedDDSWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestRootedDDSWithFSO.java index 6db7c1f..54faa6d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestRootedDDSWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestRootedDDSWithFSO.java
@@ -41,9 +41,9 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.om.OMConfigKeys; @@ -92,7 +92,7 @@ public static void init() throws Exception { // create a volume and a bucket to be used by OzoneFileSystem OzoneBucket bucket = - TestDataUtil.createVolumeAndBucket(client, getFSOBucketLayout()); + DataTestUtil.createVolumeAndBucket(client, getFSOBucketLayout()); String volumeName = bucket.getVolumeName(); volumePath = new Path(OZONE_URI_DELIMITER, volumeName); String bucketName = bucket.getName();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingServiceIntegrationTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingServiceIntegrationTest.java index ab2bb7c..f276c4b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingServiceIntegrationTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingServiceIntegrationTest.java
@@ -55,9 +55,9 @@ import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -146,7 +146,7 @@ public void setup() throws Exception { cluster.waitForClusterToBeReady(); client = cluster.newClient(); om = cluster.getOzoneManager(); - bucket1 = TestDataUtil.createVolumeAndBucket( + bucket1 = DataTestUtil.createVolumeAndBucket( client, VOLUME_NAME, BUCKET_NAME_ONE, BucketLayout.DEFAULT); } @@ -218,11 +218,11 @@ public void testMultipleSnapshotKeyReclaim() throws Exception { .setBucketLayout(BucketLayout.LEGACY) .build(); - OzoneBucket bucket2 = TestDataUtil.createBucket( + OzoneBucket bucket2 = DataTestUtil.createBucket( client, VOLUME_NAME, bucketArgs, BUCKET_NAME_TWO); // Create key1 and key2 - TestDataUtil.createKey(bucket2, "bucket2key1", CONTENT.array()); - TestDataUtil.createKey(bucket2, "bucket2key2", CONTENT.array()); + DataTestUtil.createKey(bucket2, "bucket2key1", CONTENT.array()); + DataTestUtil.createKey(bucket2, "bucket2key2", CONTENT.array()); // Create Snapshot client.getObjectStore().createSnapshot(VOLUME_NAME, BUCKET_NAME_TWO, @@ -275,7 +275,7 @@ public void testSnapshotWithFSO() throws Exception { BucketArgs bucketArgs = new BucketArgs.Builder() .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED) .build(); - OzoneBucket bucket2 = TestDataUtil.createBucket( + OzoneBucket bucket2 = DataTestUtil.createBucket( client, VOLUME_NAME, bucketArgs, BUCKET_NAME_FSO); assertTableRowCount(snapshotInfoTable, 0); @@ -286,12 +286,12 @@ public void testSnapshotWithFSO() throws Exception { om.getKeyManager().getDeletingService().suspend(); // Create 10 keys for (int i = 1; i <= 10; i++) { - TestDataUtil.createKey(bucket2, "key" + i, CONTENT.array()); + DataTestUtil.createKey(bucket2, "key" + i, CONTENT.array()); } // Create 5 keys to overwrite for (int i = 11; i <= 15; i++) { - TestDataUtil.createKey(bucket2, "key" + i, CONTENT.array()); + DataTestUtil.createKey(bucket2, "key" + i, CONTENT.array()); } // Create Directory and Sub @@ -304,7 +304,7 @@ public void testSnapshotWithFSO() throws Exception { String childDir = "/childDir" + j; client.getProxy().createDirectory(VOLUME_NAME, BUCKET_NAME_FSO, parent + childDir); - TestDataUtil.createKey(bucket2, parent + childFile, CONTENT.array()); + DataTestUtil.createKey(bucket2, parent + childFile, CONTENT.array()); } } @@ -320,7 +320,7 @@ public void testSnapshotWithFSO() throws Exception { // Overwrite 3 keys -> Moves previous version to deletedTable for (int i = 11; i <= 13; i++) { - TestDataUtil.createKey(bucket2, "key" + i, CONTENT.array()); + DataTestUtil.createKey(bucket2, "key" + i, CONTENT.array()); } assertTableRowCount(keyTable, 24); @@ -384,7 +384,7 @@ public void testSnapshotWithFSO() throws Exception { // Overwrite 2 keys for (int i = 14; i <= 15; i++) { - TestDataUtil.createKey(bucket2, "key" + i, CONTENT.array()); + DataTestUtil.createKey(bucket2, "key" + i, CONTENT.array()); } // Delete 2 more keys @@ -565,8 +565,8 @@ private synchronized void createSnapshotDataForBucket(OzoneBucket bucket) throws OmMetadataManagerImpl metadataManager = (OmMetadataManagerImpl) om.getMetadataManager(); - TestDataUtil.createKey(bucket, bucket.getName() + "key0", CONTENT.array()); - TestDataUtil.createKey(bucket, bucket.getName() + "key1", CONTENT.array()); + DataTestUtil.createKey(bucket, bucket.getName() + "key0", CONTENT.array()); + DataTestUtil.createKey(bucket, bucket.getName() + "key1", CONTENT.array()); assertTableRowCount(keyTable, 2); // Create Snapshot 1. @@ -576,8 +576,8 @@ private synchronized void createSnapshotDataForBucket(OzoneBucket bucket) throws // Overwrite bucket1key0, This is a newer version of the key which should // reclaimed as this is a different version of the key. - TestDataUtil.createKey(bucket, bucket.getName() + "key0", CONTENT.array()); - TestDataUtil.createKey(bucket, bucket.getName() + "key2", CONTENT.array()); + DataTestUtil.createKey(bucket, bucket.getName() + "key0", CONTENT.array()); + DataTestUtil.createKey(bucket, bucket.getName() + "key2", CONTENT.array()); // Key 1 cannot be reclaimed as it is still referenced by Snapshot 1. client.getProxy().deleteKey(bucket.getVolumeName(), bucket.getName(), @@ -601,8 +601,8 @@ private synchronized void createSnapshotDataForBucket(OzoneBucket bucket) throws // deletedTable when Snapshot 2 is taken. assertTableRowCount(deletedTable, 0); - TestDataUtil.createKey(bucket, bucket.getName() + "key3", CONTENT.array()); - TestDataUtil.createKey(bucket, bucket.getName() + "key4", CONTENT.array()); + DataTestUtil.createKey(bucket, bucket.getName() + "key3", CONTENT.array()); + DataTestUtil.createKey(bucket, bucket.getName() + "key4", CONTENT.array()); client.getProxy().deleteKey(bucket.getVolumeName(), bucket.getName(), bucket.getName() + "key4", false); assertTableRowCount(keyTable, 1); @@ -694,7 +694,7 @@ public void testSnapshotDeletingServiceWaitsForKeyDeletingService(boolean kdsRun UUID snap1Id = client.getObjectStore().getSnapshotInfo(volume, bucket, "snap0").getSnapshotId(); // Create snap1 - TestDataUtil.createKey(ozoneBucket, "key", CONTENT.array()); + DataTestUtil.createKey(ozoneBucket, "key", CONTENT.array()); client.getObjectStore().createSnapshot(volume, bucket, "snap1"); UUID snap2Id = client.getObjectStore().getSnapshotInfo(volume, bucket, "snap1").getSnapshotId();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotTests.java index 44131d8..40dd14d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotTests.java
@@ -88,6 +88,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.fs.FSDataOutputStream; @@ -115,10 +116,10 @@ import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksObjectUtils; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -262,7 +263,7 @@ private void init() throws Exception { cluster.waitForClusterToBeReady(); client = cluster.newClient(); // create a volume and a bucket to be used by OzoneFileSystem - ozoneBucket = TestDataUtil.createVolumeAndBucket(client, bucketLayout, null, createLinkedBucket); + ozoneBucket = DataTestUtil.createVolumeAndBucket(client, bucketLayout, null, createLinkedBucket); if (createLinkedBucket) { this.linkedBuckets.put(ozoneBucket.getName(), ozoneBucket.getSourceBucket()); } @@ -283,7 +284,7 @@ private void createBucket(OzoneVolume volume, String bucketVal) throws IOExcepti if (createLinkedBucket) { String sourceBucketName = linkedBuckets.computeIfAbsent(bucketVal, (k) -> bucketVal + counter.incrementAndGet()); volume.createBucket(sourceBucketName); - TestDataUtil.createLinkedBucket(client, volume.getName(), sourceBucketName, bucketVal); + DataTestUtil.createLinkedBucket(client, volume.getName(), sourceBucketName, bucketVal); this.linkedBuckets.put(bucketVal, sourceBucketName); } else { volume.createBucket(bucketVal); @@ -3261,18 +3262,21 @@ public void testSnapshotDiffWithCreateMultipartKeys() throws Exception { try (OzoneOutputStream stream = bucket.createMultipartKey( regularPartsKey, regularPart.length, 1, regularMpuInfo.getUploadID())) { stream.write(regularPart); + stream.getMetadata().put(OzoneConsts.ETAG, DigestUtils.md5Hex(regularPart)); } byte[] streamPart = "stream data".getBytes(UTF_8); try (OzoneDataStreamOutput streamOut = bucket.createMultipartStreamKey( streamPartsKey, streamPart.length, 1, streamMpuInfo.getUploadID())) { streamOut.write(streamPart); + streamOut.getMetadata().put(OzoneConsts.ETAG, DigestUtils.md5Hex(streamPart)); } byte[] mixedPart = "mixed data".getBytes(UTF_8); try (OzoneOutputStream mixedStream = bucket.createMultipartKey( mixedPartsKey, mixedPart.length, 1, mixedMpuInfo.getUploadID())) { mixedStream.write(mixedPart); + mixedStream.getMetadata().put(OzoneConsts.ETAG, DigestUtils.md5Hex(mixedPart)); } assertEquals(1, @@ -3322,14 +3326,17 @@ public void testSnapshotDiffWithAbortMultipartUpload() throws Exception { try (OzoneOutputStream part1Stream = bucket.createMultipartKey( partialAbortKey, part1Data.length, 1, partialInfo.getUploadID())) { part1Stream.write(part1Data); + part1Stream.getMetadata().put(OzoneConsts.ETAG, DigestUtils.md5Hex(part1Data)); } try (OzoneOutputStream part2Stream = bucket.createMultipartKey( partialAbortKey, part2Data.length, 2, partialInfo.getUploadID())) { part2Stream.write(part2Data); + part2Stream.getMetadata().put(OzoneConsts.ETAG, DigestUtils.md5Hex(part2Data)); } try (OzoneDataStreamOutput part3Stream = bucket.createMultipartStreamKey( partialAbortKey, part3Data.length, 3, partialInfo.getUploadID())) { part3Stream.write(part3Data); + part3Stream.getMetadata().put(OzoneConsts.ETAG, DigestUtils.md5Hex(part3Data)); } OzoneMultipartUploadPartListParts partsList = bucket.listParts( @@ -3348,10 +3355,12 @@ public void testSnapshotDiffWithAbortMultipartUpload() throws Exception { try (OzoneOutputStream stream = bucket.createMultipartKey( multiAbortKey1, part1Data.length, 1, multiInfo1.getUploadID())) { stream.write(part1Data); + stream.getMetadata().put(OzoneConsts.ETAG, DigestUtils.md5Hex(part1Data)); } try (OzoneDataStreamOutput stream = bucket.createMultipartStreamKey( multiAbortKey2, part2Data.length, 1, multiInfo2.getUploadID())) { stream.write(part2Data); + stream.getMetadata().put(OzoneConsts.ETAG, DigestUtils.md5Hex(part2Data)); } bucket.abortMultipartUpload(multiAbortKey1, multiInfo1.getUploadID()); @@ -3398,10 +3407,12 @@ public void testSnapshotDiffWithCompleteInvisibleMPULifecycle() throws Exception try (OzoneOutputStream stream = bucket.createMultipartKey( mpuKey1, regularData1.length, 1, mpuInfo1.getUploadID())) { stream.write(regularData1); + stream.getMetadata().put(OzoneConsts.ETAG, DigestUtils.md5Hex(regularData1)); } try (OzoneOutputStream stream = bucket.createMultipartKey( mpuKey1, regularData2.length, 2, mpuInfo1.getUploadID())) { stream.write(regularData2); + stream.getMetadata().put(OzoneConsts.ETAG, DigestUtils.md5Hex(regularData2)); } byte[] streamData1 = "Stream multipart data 1".getBytes(UTF_8); @@ -3410,10 +3421,12 @@ public void testSnapshotDiffWithCompleteInvisibleMPULifecycle() throws Exception try (OzoneDataStreamOutput stream = bucket.createMultipartStreamKey( mpuKey2, streamData1.length, 1, mpuInfo2.getUploadID())) { stream.write(streamData1); + stream.getMetadata().put(OzoneConsts.ETAG, DigestUtils.md5Hex(streamData1)); } try (OzoneDataStreamOutput stream = bucket.createMultipartStreamKey( mpuKey2, streamData2.length, 2, mpuInfo2.getUploadID())) { stream.write(streamData2); + stream.getMetadata().put(OzoneConsts.ETAG, DigestUtils.md5Hex(streamData2)); } @@ -3423,10 +3436,12 @@ public void testSnapshotDiffWithCompleteInvisibleMPULifecycle() throws Exception try (OzoneOutputStream stream = bucket.createMultipartKey( mpuKey3, mixedRegular.length, 1, mpuInfo3.getUploadID())) { stream.write(mixedRegular); + stream.getMetadata().put(OzoneConsts.ETAG, DigestUtils.md5Hex(mixedRegular)); } try (OzoneDataStreamOutput stream = bucket.createMultipartStreamKey( mpuKey3, mixedStream.length, 2, mpuInfo3.getUploadID())) { stream.write(mixedStream); + stream.getMetadata().put(OzoneConsts.ETAG, DigestUtils.md5Hex(mixedStream)); } assertEquals(2, @@ -3470,6 +3485,7 @@ private void completeSinglePartMPU(OzoneBucket bucket, String keyName, String da byte[] partData = createLargePartData(data, MIN_PART_SIZE); OzoneOutputStream partStream = bucket.createMultipartKey(keyName, partData.length, 1, uploadId); partStream.write(partData); + partStream.getMetadata().put(OzoneConsts.ETAG, DigestUtils.md5Hex(partData)); partStream.close(); OzoneMultipartUploadPartListParts partsList = bucket.listParts(keyName, uploadId, 0, 100); @@ -3491,6 +3507,7 @@ private void completeMultiplePartMPU( try (OzoneOutputStream partStream = bucket.createMultipartKey( keyName, partData.length, partNum, uploadId)) { partStream.write(partData); + partStream.getMetadata().put(OzoneConsts.ETAG, DigestUtils.md5Hex(partData)); } } @@ -3514,12 +3531,14 @@ private void completeMixedPartMPU( try (OzoneOutputStream partStream = bucket.createMultipartKey( keyName, part1Data.length, 1, uploadId)) { partStream.write(part1Data); + partStream.getMetadata().put(OzoneConsts.ETAG, DigestUtils.md5Hex(part1Data)); } byte[] part2Data = createLargePartData(streamData, MIN_PART_SIZE); try (OzoneDataStreamOutput partStream = bucket.createMultipartStreamKey( keyName, part2Data.length, 2, uploadId)) { partStream.write(part2Data); + partStream.getMetadata().put(OzoneConsts.ETAG, DigestUtils.md5Hex(part2Data)); } OzoneMultipartUploadPartListParts partsList = bucket.listParts(keyName, uploadId, 0, 2); @@ -3547,6 +3566,7 @@ private void completeMPUWithReplication( try (OzoneOutputStream partStream = bucket.createMultipartKey( keyName, partData.length, 1, uploadId)) { partStream.write(partData); + partStream.getMetadata().put(OzoneConsts.ETAG, DigestUtils.md5Hex(partData)); } OzoneMultipartUploadPartListParts partsList = bucket.listParts(keyName, uploadId, 0, 1); @@ -3566,6 +3586,7 @@ private void completeMPUWithMetadata(OzoneBucket bucket, String keyName, byte[] partData = createLargePartData("MPU with metadata and tags", MIN_PART_SIZE); OzoneOutputStream partStream = bucket.createMultipartKey(keyName, partData.length, 1, uploadId); partStream.write(partData); + partStream.getMetadata().put(OzoneConsts.ETAG, DigestUtils.md5Hex(partData)); partStream.close(); OzoneMultipartUploadPartListParts partsList = bucket.listParts(keyName, uploadId, 0, 100);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystem.java index 9645137..3d7d277 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystem.java
@@ -63,8 +63,8 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -132,7 +132,7 @@ public void setupFsClient() throws IOException { writeClient = objectStore.getClientProxy().getOzoneManagerClient(); ozoneManager = cluster().getOzoneManager(); - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client, bucketLayout, null, createLinkedBuckets); + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client, bucketLayout, null, createLinkedBuckets); if (createLinkedBuckets) { linkedBucketMaps.put(bucket.getName(), bucket.getSourceBucket()); } @@ -320,7 +320,7 @@ private void createKeys(OzoneBucket ozoneBucket, List<String> keys) private void createKey(OzoneBucket ozoneBucket, String key, int length) throws Exception { - byte[] input = TestDataUtil.createStringKey(ozoneBucket, key, length); + byte[] input = DataTestUtil.createStringKey(ozoneBucket, key, length); // Read the key with given key name. readkey(ozoneBucket, key, length, input); }
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerHASnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerHASnapshot.java index cd461b7..05dabc8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerHASnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerHASnapshot.java
@@ -43,10 +43,10 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.RDBCheckpointUtils; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -90,7 +90,7 @@ public static void staticInit() throws Exception { cluster.waitForClusterToBeReady(); client = cluster.newClient(); store = client.getObjectStore(); - ozoneBucket = TestDataUtil.createVolumeAndBucket(client); + ozoneBucket = DataTestUtil.createVolumeAndBucket(client); volumeName = ozoneBucket.getVolumeName(); bucketName = ozoneBucket.getName(); } @@ -235,7 +235,7 @@ public void testSnapshotChainManagerRestore() throws Exception { // Create 10 buckets and initialize snapshot name lists. for (int i = 0; i < 10; i++) { - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client); + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client); ozoneBuckets.add(bucket); volumeNames.add(bucket.getVolumeName()); bucketNames.add(bucket.getName());
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotBackgroundServices.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotBackgroundServices.java index 5e3f49e..7dd31fd 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotBackgroundServices.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotBackgroundServices.java
@@ -18,12 +18,12 @@ package org.apache.hadoop.ozone.om.snapshot; import static java.util.stream.Collectors.toSet; +import static org.apache.hadoop.ozone.DataTestUtil.readFully; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_COMPACTION_DAG_MAX_TIME_ALLOWED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPSHOT_COMPACTION_DAG_PRUNE_DAEMON_RUN_INTERVAL; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.TestDataUtil.readFully; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath; import static org.apache.hadoop.ozone.om.TestOzoneManagerHAWithStoppedNodes.createKey;
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDirectoryCleaningService.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDirectoryCleaningService.java index a34e11e..d68d6f9 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDirectoryCleaningService.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDirectoryCleaningService.java
@@ -43,9 +43,9 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.om.OMConfigKeys; @@ -98,7 +98,7 @@ public static void init() throws Exception { client = cluster.newClient(); // create a volume and a bucket to be used by OzoneFileSystem - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client, + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client, BucketLayout.FILE_SYSTEM_OPTIMIZED); volumeName = bucket.getVolumeName(); bucketName = bucket.getName(); @@ -305,7 +305,7 @@ public void testSnapshotDiffBeforeAndAfterDeepCleaning() throws Exception { String volume = "vol-" + counter.incrementAndGet(); String bucket = "buc-" + counter.incrementAndGet(); // create a volume and a bucket to be used by OzoneFileSystem - OzoneBucket volBucket = TestDataUtil.createVolumeAndBucket(client, volume, bucket, + OzoneBucket volBucket = DataTestUtil.createVolumeAndBucket(client, volume, bucket, BucketLayout.FILE_SYSTEM_OPTIMIZED); volBucket.createDirectory("dir1/dir2/dir3/dir4"); cluster.getOzoneManager().getKeyManager().getDirDeletingService().suspend();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneContainerUpgradeShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneContainerUpgradeShell.java index 215e7a5..9572963 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneContainerUpgradeShell.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneContainerUpgradeShell.java
@@ -50,10 +50,10 @@ import org.apache.hadoop.hdds.utils.db.CodecTestUtil; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksObjectMetrics; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneTestUtils; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; @@ -150,8 +150,8 @@ private static ContainerInfo writeKeyAndCloseContainer() throws Exception { } private static void writeKey(String keyName) throws IOException { - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client, VOLUME_NAME, BUCKET_NAME); - TestDataUtil.createKey(bucket, keyName, "test".getBytes(StandardCharsets.UTF_8)); + OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client, VOLUME_NAME, BUCKET_NAME); + DataTestUtil.createKey(bucket, keyName, "test".getBytes(StandardCharsets.UTF_8)); } private static ContainerInfo closeContainerForKey(String keyName)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugReplicasVerify.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugReplicasVerify.java index 32f7d1a..70d0ca3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugReplicasVerify.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugReplicasVerify.java
@@ -17,7 +17,7 @@ package org.apache.hadoop.ozone.shell; -import static org.apache.hadoop.ozone.TestDataUtil.createKeys; +import static org.apache.hadoop.ozone.DataTestUtil.createKeys; import static org.apache.hadoop.ozone.container.ContainerTestHelper.corruptFile; import static org.apache.hadoop.ozone.container.ContainerTestHelper.truncateFile; import static org.apache.ozone.test.GenericTestUtils.setLogLevel;
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java index 0616906..0207cdd 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java
@@ -45,8 +45,8 @@ import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.OzoneTestUtils; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneSnapshot; import org.apache.hadoop.ozone.debug.OzoneDebug; @@ -189,9 +189,9 @@ private void writeKey(String volumeName, String bucketName, repConfig = ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE); } - TestDataUtil.createVolumeAndBucket(client, volumeName, bucketName, + DataTestUtil.createVolumeAndBucket(client, volumeName, bucketName, layout); - TestDataUtil.createKey( + DataTestUtil.createKey( client.getObjectStore().getVolume(volumeName).getBucket(bucketName), keyName, repConfig, "test".getBytes(StandardCharsets.UTF_8)); }
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java index bc8936b..c132f96 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java
@@ -41,8 +41,8 @@ import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.DataTestUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneKeyDetails; @@ -100,7 +100,7 @@ void init(@TempDir Path path) throws Exception { conf.unset(key); } - TestDataUtil.createVolume(client, VOLUME_NAME); + DataTestUtil.createVolume(client, VOLUME_NAME); volume = client.getObjectStore().getVolume(VOLUME_NAME); }
diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index f09020b..bccc6d1 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -1770,6 +1770,7 @@ message MultipartInfoInitiateRequest { required KeyArgs keyArgs = 1; + optional uint32 schemaVersion = 2; }
diff --git a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/FixedHostMapping.java b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/FixedHostMapping.java new file mode 100644 index 0000000..801af9f --- /dev/null +++ b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/FixedHostMapping.java
@@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; +import org.apache.hadoop.net.CachedDNSToSwitchMapping; +import org.apache.hadoop.net.DNSToSwitchMapping; +import org.apache.hadoop.net.NetworkTopology; + +/** + * A {@link CachedDNSToSwitchMapping} implementation that resolves hostnames + * to rack locations using a statically configured map, bypassing DNS lookups. + * + * <p>This is intended for use in test environments (e.g. {@code MiniOzoneCluster}) + * where DataNode hostnames may be synthetic or unresolvable via DNS. The standard + * {@link CachedDNSToSwitchMapping} performs DNS normalization before rack resolution, + * which can cause synthetic hostnames to be incorrectly resolved to a real IP address, + * leading to rack mapping failures. This class avoids that by resolving directly + * against the registered hostname. + * + * <p>The mapping is stored in a JVM-wide static map. Callers must invoke + * {@link #addNode(String, String)} before cluster startup to register hostname-to-rack + * entries, and should call {@link #clear()} after each test to avoid cross-test pollution. + * + * <p>Usage: + * <pre>{@code + * FixedHostMapping.addNode("dn-0.test", "/rack1"); + * FixedHostMapping.addNode("dn-1.test", "/rack1"); + * FixedHostMapping.addNode("dn-2.test", "/rack2"); + * + * conf.setClass( + * CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, + * FixedHostMapping.class, + * DNSToSwitchMapping.class); + * }</pre> + */ +public class FixedHostMapping extends CachedDNSToSwitchMapping { + + private static final Map<String, String> RACK_MAP = new ConcurrentHashMap<>(); + + /** + * Constructs a {@code FixedHostMapping} with a no-op raw mapping. + * The raw mapping is unused since {@link #resolve(List)} is fully overridden. + */ + public FixedHostMapping() { + super(new NoOpMapping()); + } + + /** + * Constructs a {@code FixedHostMapping} with the given raw mapping. + * The raw mapping is unused since {@link #resolve(List)} is fully overridden, + * but is accepted to satisfy {@link CachedDNSToSwitchMapping} constructor requirements. + * + * @param rawMapping the raw DNS mapping (not used for resolution) + */ + public FixedHostMapping(DNSToSwitchMapping rawMapping) { + super(rawMapping); + } + + /** + * Registers a hostname-to-rack mapping entry. + * Must be called before cluster startup for the mapping to take effect during + * DataNode registration. + * + * @param host the DataNode hostname as it will appear in {@link #resolve(List)} + * @param rack the rack path (e.g. {@code "/rack1"}) + */ + public static void addNode(String host, String rack) { + RACK_MAP.put(host, rack); + } + + /** + * Clears all registered hostname-to-rack mappings. + * Should be called in test teardown (e.g. {@code @AfterEach}) to prevent + * cross-test pollution of the JVM-wide static map. + */ + public static void clear() { + RACK_MAP.clear(); + } + + /** + * Resolves a list of hostnames to their rack locations using the static map. + * Hostnames not present in the map are assigned {@link NetworkTopology#DEFAULT_RACK}. + * Unlike the parent class, this method does not perform DNS normalization. + * + * @param names the list of hostnames to resolve + * @return a list of rack paths in the same order as the input + */ + @Override + public List<String> resolve(List<String> names) { + return names.stream() + .map(name -> RACK_MAP.getOrDefault(name, NetworkTopology.DEFAULT_RACK)) + .collect(Collectors.toList()); + } + + /** + * No-op: this implementation does not maintain a cache. + */ + @Override + public void reloadCachedMappings() { + } + + /** + * No-op: this implementation does not maintain a cache. + * + * @param names the hostnames whose cached mappings should be reloaded (ignored) + */ + @Override + public void reloadCachedMappings(List<String> names) { + } + + /** + * A no-op {@link DNSToSwitchMapping} used as a placeholder raw mapping. + * All hostnames are mapped to {@link NetworkTopology#DEFAULT_RACK}. + * This is never invoked during normal resolution since {@link FixedHostMapping#resolve(List)} + * is fully overridden. + */ + private static class NoOpMapping implements DNSToSwitchMapping { + + @Override + public List<String> resolve(List<String> names) { + return names.stream() + .map(n -> NetworkTopology.DEFAULT_RACK) + .collect(Collectors.toList()); + } + + @Override + public void reloadCachedMappings() { + } + + @Override + public void reloadCachedMappings(List<String> names) { + } + } +}
diff --git a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneCluster.java index d768e44..8765c2a 100644 --- a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneCluster.java +++ b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -19,9 +19,11 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.UUID; import java.util.concurrent.TimeoutException; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -263,6 +265,8 @@ abstract class Builder { protected CertificateClient certClient; protected SecretKeyClient secretKeyClient; protected DatanodeFactory dnFactory = UniformDatanodesFactory.newBuilder().build(); + protected String[] racks; + protected String[] hosts; private final List<Service> services = new ArrayList<>(); protected Builder(OzoneConfiguration conf) { @@ -289,6 +293,15 @@ protected void prepareForNextBuild() { conf.unset(OMConfigKeys.OZONE_OM_DB_DIRS); conf.unset(OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_DB_DIR); + // dn rack configs + if (racks != null) { + conf.unset(CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY); + conf.unset(HddsConfigKeys.HDDS_DATANODE_USE_DN_HOSTNAME); + conf.unset("hadoop.configured.node.mapping"); + racks = null; + hosts = null; + } + setClusterId(); } @@ -369,6 +382,43 @@ public Builder setDatanodeFactory(DatanodeFactory factory) { return this; } + /** + * Sets the rack location for each datanode. Each entry is a rack path + * such as {@code "/rack0"}. The length of the array must match the + * number of datanodes. + * + * @param racks rack path per datanode + * @return this Builder + */ + public Builder setRacks(String[] racks) { + this.racks = Arrays.copyOf(racks, racks.length); + return this; + } + + /** + * Sets the hostname for each datanode. When used together with + * {@link #setRacks}, the hostnames are used as keys in the + * {@code StaticMapping} instead of the default synthetic names + * ({@code "dn-0"}, {@code "dn-1"}, …). The length of the array must + * match the number of datanodes. + * + * @param hosts hostname per datanode + * @return this Builder + */ + public Builder setHosts(String[] hosts) { + this.hosts = Arrays.copyOf(hosts, hosts.length); + return this; + } + + protected void validateDatanodeConfiguration() { + if (racks != null && racks.length != numOfDatanodes) { + throw new IllegalArgumentException("Number of racks must match the number of datanodes"); + } + if (hosts != null && hosts.length != numOfDatanodes) { + throw new IllegalArgumentException("Number of hosts must match the number of datanodes"); + } + } + public Builder addService(Service service) { services.add(service); return this;
diff --git a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java index b0e2c1e..e13719d 100644 --- a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java +++ b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -39,12 +39,14 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import org.apache.commons.io.FileUtils; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.DatanodeID; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.HddsTestUtils; import org.apache.hadoop.hdds.scm.ScmConfigKeys; @@ -54,6 +56,7 @@ import org.apache.hadoop.hdds.scm.node.NodeStatus; import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; +import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.scm.proxy.SCMClientConfig; import org.apache.hadoop.hdds.scm.proxy.SCMContainerLocationFailoverProxyProvider; @@ -64,14 +67,17 @@ import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; +import org.apache.hadoop.hdds.utils.HddsServerUtil; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.CodecBuffer; import org.apache.hadoop.hdds.utils.db.CodecTestUtil; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksObjectMetrics; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.net.DNSToSwitchMapping; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.common.Storage.StorageState; +import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; import org.apache.hadoop.ozone.container.common.utils.ContainerCache; import org.apache.hadoop.ozone.container.common.utils.DatanodeStoreCache; import org.apache.hadoop.ozone.om.OMConfigKeys; @@ -505,6 +511,7 @@ public Builder(OzoneConfiguration conf) { @Override public MiniOzoneCluster build() throws IOException { + validateDatanodeConfiguration(); DefaultMetricsSystem.setMiniClusterMode(true); DatanodeStoreCache.setMiniClusterMode(); initializeConfiguration(); @@ -527,6 +534,12 @@ public MiniOzoneCluster build() throws IOException { cluster.startHddsDatanodes(); } + // Recreate the Ratis pipeline to prevent imbalanced node placement across racks + // caused by asynchronous DN registration. + if (racks != null && startDataNodes) { + resetPipelinesForRackAwareness(cluster); + } + prepareForNextBuild(); return cluster; } catch (Exception ex) { @@ -560,6 +573,33 @@ protected void setClients(OzoneManager om) throws IOException { } /** + * Waits for all DNs to be healthy, then removes any pipelines that + * were created before the full rack topology was visible, and creates one + * fresh rack-aware pipeline directly (bypassing the background timer). + */ + private void resetPipelinesForRackAwareness(MiniOzoneClusterImpl cluster) + throws IOException { + try { + cluster.waitForClusterToBeReady(); + } catch (TimeoutException | InterruptedException e) { + throw new IOException( + "Timed out waiting for rack-aware cluster to be ready", e); + } + RatisReplicationConfig threeWay = + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE); + PipelineManager pm = + cluster.getStorageContainerManager().getPipelineManager(); + for (Pipeline p : pm.getPipelines(threeWay)) { + if (!p.isClosed()) { + pm.closePipeline(p.getId()); + } + pm.deletePipeline(p.getId()); + } + + pm.createPipeline(threeWay); + } + + /** * Initializes the configuration required for starting MiniOzoneCluster. */ protected void initializeConfiguration() throws IOException { @@ -577,6 +617,40 @@ protected void initializeConfiguration() throws IOException { // pipeline. conf.setInt(HddsConfigKeys.HDDS_SCM_SAFEMODE_MIN_DATANODE, numOfDatanodes >= 3 ? 3 : 1); + + configureHostAndRackTopology(); + } + + private void configureHostAndRackTopology() throws IOException { + FixedHostMapping.clear(); + if (racks == null && hosts == null) { + return; + } + + conf.setBoolean(HddsConfigKeys.HDDS_DATANODE_USE_DN_HOSTNAME, true); + + if (hosts == null) { + hosts = new String[racks.length]; + for (int i = 0; i < racks.length; i++) { + hosts[i] = "host" + i + ".foo.com"; + } + } + + if (racks != null) { + + if (hosts.length != racks.length) { + throw new IllegalArgumentException( + "The length of hosts [" + hosts.length + + "] must match the length of racks [" + racks.length + "]."); + } + + conf.setClass(CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, + FixedHostMapping.class, DNSToSwitchMapping.class); + + for (int i = 0; i < racks.length; i++) { + FixedHostMapping.addNode(hosts[i], racks[i]); + } + } } void removeConfiguration() { @@ -698,14 +772,32 @@ protected List<HddsDatanodeService> createHddsDatanodes() for (int i = 0; i < numOfDatanodes; i++) { OzoneConfiguration dnConf = dnFactory.apply(conf); + if (hosts != null) { + dnConf.set(HddsConfigKeys.HDDS_DATANODE_HOST_NAME_KEY, hosts[i]); + } + // Bypass InetAddress.getName() resolution for custom hostnames by starting DN via YAML. + confDatanodeViaYaml(dnConf); HddsDatanodeService datanode = new HddsDatanodeService(NO_ARGS); + dnConf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, conf.getStrings(ScmConfigKeys.OZONE_SCM_NAMES)); datanode.setConfiguration(dnConf); hddsDatanodes.add(datanode); } + return hddsDatanodes; } + private void confDatanodeViaYaml(OzoneConfiguration dnConf) throws IOException { + DatanodeDetails datanodeDetails = DatanodeDetails.newBuilder() + .setID(DatanodeID.randomID()) + .setHostName(dnConf.get(HddsConfigKeys.HDDS_DATANODE_HOST_NAME_KEY)) + .setIpAddress("127.0.0.1") + .build(); + datanodeDetails.setNetworkName(datanodeDetails.getUuidString()); + String dnFilePath = HddsServerUtil.getDatanodeIdFilePath(dnConf); + ContainerUtils.writeDatanodeDetailsTo(datanodeDetails, new File(dnFilePath), dnConf); + } + protected void configureSCM(boolean isHA) throws IOException { conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, localhostWithFreePort());
diff --git a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java index 303a16a..507777d 100644 --- a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java +++ b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
@@ -464,6 +464,7 @@ public Builder setSCMServiceId(String serviceId) { @Override public MiniOzoneHAClusterImpl build() throws IOException { + validateDatanodeConfiguration(); if (numOfActiveOMs > numOfOMs) { throw new IllegalArgumentException("Number of active OMs cannot be " + "more than the total number of OMs");
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index 214d738..4077ee0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java
@@ -23,6 +23,8 @@ import java.util.List; import java.util.Map; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.scm.net.NetworkTopology; import org.apache.hadoop.hdds.utils.BackgroundService; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; @@ -370,4 +372,27 @@ DeleteKeysResult getPendingDeletionSubFiles(long volumeId, long bucketId, OmKeyI * @return Background service. */ KeyLifecycleService getKeyLifecycleService(); + + /** + * Sort the datanodes of a write pipeline by network-topology distance to the + * client, using OM's locally cached cluster map. Unlike the read-path sort, + * the original order is preserved when the client cannot be resolved, because + * the first node is used as the streaming-write primary. + * + * @param nodes the pipeline nodes to sort + * @param clientMachine client address (IP or hostname) + * @param clusterMap OM's cached cluster map used to resolve topology distance + * @return nodes sorted nearest-first, or the original {@code nodes} list + * instance unchanged when sorting is skipped (client unresolved or stale + * topology); callers may use reference equality to detect a skipped sort + */ + List<? extends DatanodeDetails> sortDatanodesForWrite( + List<? extends DatanodeDetails> nodes, String clientMachine, NetworkTopology clusterMap); + + /** + * @return true if OM should sort the streaming-write pipeline locally + * ({@code ozone.om.block.write.sort.datanodes.enabled}); false to leave + * the sort to SCM. + */ + boolean isSortDatanodesForWriteEnabled(); }
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 6ecfe65..f24a5c4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -123,7 +123,6 @@ import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion; import org.apache.hadoop.fs.FileEncryptionInfo; -import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -132,6 +131,7 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; import org.apache.hadoop.hdds.scm.net.InnerNode; +import org.apache.hadoop.hdds.scm.net.NetworkTopology; import org.apache.hadoop.hdds.scm.net.Node; import org.apache.hadoop.hdds.scm.net.NodeImpl; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; @@ -2262,7 +2262,12 @@ private void sortDatanodes(String clientMachine, List<OmKeyInfo> keyInfos) { List<? extends DatanodeDetails> sortedNodes = sortedPipelines.get(uuidSet); if (sortedNodes == null) { sortedNodes = sortDatanodes(nodes, clientMachine); - if (sortedNodes != null) { + // Cache only a freshly sorted order, not an input list returned + // unchanged when no sort happens: that order is per-pipeline and must + // not be reused for another pipeline with the same node set. The read + // sort always returns a new list, so this never skips caching here; it + // keeps the pattern identical to the write path. + if (sortedNodes != null && sortedNodes != nodes) { sortedPipelines.put(uuidSet, sortedNodes); } } else if (LOG.isDebugEnabled()) { @@ -2280,32 +2285,87 @@ private void sortDatanodes(String clientMachine, List<OmKeyInfo> keyInfos) { @VisibleForTesting public List<? extends DatanodeDetails> sortDatanodes(List<? extends DatanodeDetails> nodes, String clientMachine) { - final Node client = getClientNode(clientMachine, nodes); - return ozoneManager.getClusterMap() - .sortByDistanceCost(client, nodes, nodes.size()); + final NetworkTopology clusterMap = ozoneManager.getClusterMap(); + final Node client = getClientNode(clientMachine, nodes, clusterMap); + return clusterMap.sortByDistanceCost(client, nodes, nodes.size()); + } + + @Override + public List<? extends DatanodeDetails> sortDatanodesForWrite( + List<? extends DatanodeDetails> nodes, String clientMachine, NetworkTopology clusterMap) { + Preconditions.checkArgument(!StringUtils.isEmpty(clientMachine), + "clientMachine is empty"); + Objects.requireNonNull(clusterMap, "clusterMap is null"); + return captureLatencyNs( + metrics.getAllocateBlockSortDatanodesLatencyNs(), () -> { + final Node client = getClientNode(clientMachine, nodes, clusterMap); + if (client == null) { + // Preserve pipeline order for writes: the first node is the write + // primary, so do not shuffle when the client cannot be resolved. + return nodes; + } + return sortByClusterMapDistance(clusterMap, client, nodes); + }); + } + + @Override + public boolean isSortDatanodesForWriteEnabled() { + return ozoneManager.getConfig().isSortDatanodesForWriteEnabled(); + } + + /** + * Sort a pipeline's nodes by topology distance to the client. The nodes come + * from SCM over RPC, so they are deserialized {@link DatanodeDetails} with no + * parent/level: the topology treats them as unknown (distance + * {@link Integer#MAX_VALUE}) and the order comes out random. Look each node + * up in OM's cluster map to get the topology-linked instance, sort those, + * then map the order back to the original nodes. + */ + private List<? extends DatanodeDetails> sortByClusterMapDistance( + NetworkTopology clusterMap, Node client, + List<? extends DatanodeDetails> nodes) { + final List<Node> topologyNodes = new ArrayList<>(nodes.size()); + final Map<String, DatanodeDetails> nodeByPath = new HashMap<>(); + for (DatanodeDetails node : nodes) { + final Node resolved = clusterMap.getNode(node.getNetworkFullPath()); + if (resolved == null) { + return nodes; + } + topologyNodes.add(resolved); + nodeByPath.put(resolved.getNetworkFullPath(), node); + } + final List<Node> sorted = + clusterMap.sortByDistanceCost(client, topologyNodes, topologyNodes.size()); + final List<DatanodeDetails> result = new ArrayList<>(sorted.size()); + for (Node node : sorted) { + result.add(nodeByPath.get(node.getNetworkFullPath())); + } + return result; } private Node getClientNode(String clientMachine, - List<? extends DatanodeDetails> nodes) { - List<DatanodeDetails> matchingNodes = new ArrayList<>(); - boolean useHostname = ozoneManager.getConfiguration().getBoolean( - HddsConfigKeys.HDDS_DATANODE_USE_DN_HOSTNAME, - HddsConfigKeys.HDDS_DATANODE_USE_DN_HOSTNAME_DEFAULT); + List<? extends DatanodeDetails> nodes, NetworkTopology clusterMap) { for (DatanodeDetails node : nodes) { - if ((useHostname ? node.getHostName() : node.getIpAddress()).equals( - clientMachine)) { - matchingNodes.add(node); + // Match by either IP or hostname, like SCM's getNodesByAddress. clientMachine + // may be a hostname on the read path; the streaming-write remoteAddress is + // typically an IP. Matching both covers use.datanode.hostname either way. + if (clientMachine.equals(node.getIpAddress()) + || clientMachine.equals(node.getHostName())) { + // The pipeline nodes are RPC-deserialized and not linked into OM's + // cluster map; prefer the map's instance so distance can be computed. + final Node resolved = clusterMap.getNode(node.getNetworkFullPath()); + return resolved != null ? resolved : node; } } - return !matchingNodes.isEmpty() ? matchingNodes.get(0) : - getOtherNode(clientMachine); + return getOtherNode(clientMachine, clusterMap); } - private Node getOtherNode(String clientMachine) { + private Node getOtherNode(String clientMachine, + NetworkTopology clusterMap) { try { String clientLocation = resolveNodeLocation(clientMachine); if (clientLocation != null) { - Node rack = ozoneManager.getClusterMap().getNode(clientLocation); + Node rack = clusterMap.getNode(clientLocation); if (rack instanceof InnerNode) { return new NodeImpl(clientMachine, clientLocation, (InnerNode) rack, rack.getLevel() + 1,
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java index e142e4c..3f965ba 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java
@@ -67,6 +67,9 @@ public class OMPerformanceMetrics { @Metric(about = "Sort datanodes latency in getKeyInfo") private MutableRate getKeyInfoSortDatanodesLatencyNs; + @Metric(about = "Sort datanodes latency in allocateBlock (streaming write)") + private MutableRate allocateBlockSortDatanodesLatencyNs; + @Metric(about = "resolveBucketLink latency in getKeyInfo") private MutableRate getKeyInfoResolveBucketLatencyNs; @@ -246,6 +249,10 @@ MutableRate getGetKeyInfoSortDatanodesLatencyNs() { return getKeyInfoSortDatanodesLatencyNs; } + MutableRate getAllocateBlockSortDatanodesLatencyNs() { + return allocateBlockSortDatanodesLatencyNs; + } + public void setForceContainerCacheRefresh(boolean value) { forceContainerCacheRefresh.add(value ? 1L : 0L); }
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index f31815b..99b1c18 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -1559,8 +1559,13 @@ public List<ExpiredMultipartUploadsBucket> getExpiredMultipartUploads( expiredMPUs.get(mapKey) .addMultipartUploads(builder.setName(dbMultipartInfoKey) .build()); - numParts += omMultipartKeyInfo.getPartKeyInfoMap().size(); - // TODO: Add the expired part handling from the new table when the complete flow is done + + if (omMultipartKeyInfo.getSchemaVersion() + == OmMultipartKeyInfo.SPLIT_PARTS_TABLE_SCHEMA_VERSION) { + numParts += OMMultipartUploadUtils.countParts(this, expiredMultipartUpload.getUploadId()); + } else { + numParts += omMultipartKeyInfo.getPartKeyInfoMap().size(); + } } }
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index b918383..2db20c5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.om; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Objects.requireNonNull; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED; @@ -1350,6 +1351,11 @@ public void setScmTopologyClient( } public NetworkTopology getClusterMap() { + return requireNonNull(getClusterMapAllowNull(), + "OM topology cache has not been initialized yet."); + } + + public NetworkTopology getClusterMapAllowNull() { return scmTopologyClient.getClusterMap(); } @@ -3005,8 +3011,18 @@ public List<OmBucketInfo> listBuckets(String volumeName, String startKey, volumeName, null, null); } metrics.incNumBucketLists(); - return bucketManager.listBuckets(volumeName, + List<OmBucketInfo> buckets = bucketManager.listBuckets(volumeName, startKey, prefix, maxNumOfBuckets, hasSnapshot); + Map<Pair<String, String>, OmBucketInfo> resolvedSourceCache = new HashMap<>(); + for (int i = 0; i < buckets.size(); i++) { + try { + buckets.set(i, enrichLinkBucketInfo(buckets.get(i), resolvedSourceCache)); + } catch (IOException e) { + LOG.debug("Failed to enrich listBuckets entry for {}/{}; returning raw entry", + volumeName, buckets.get(i).getBucketName(), e); + } + } + return buckets; } catch (IOException ex) { metrics.incNumBucketListFails(); auditSuccess = false; @@ -3022,6 +3038,62 @@ public List<OmBucketInfo> listBuckets(String volumeName, String startKey, } /** + * For link buckets, follows the link chain and overlays the source bucket's + * operational properties onto the link's {@link OmBucketInfo}. Non-link + * buckets and dangling links are returned unchanged. + */ + private OmBucketInfo enrichLinkBucketInfo(OmBucketInfo bucketInfo) + throws IOException { + return enrichLinkBucketInfo(bucketInfo, null); + } + + private OmBucketInfo enrichLinkBucketInfo( + OmBucketInfo bucketInfo, + Map<Pair<String, String>, OmBucketInfo> resolvedSourceCache) + throws IOException { + if (!bucketInfo.isLink()) { + return bucketInfo; + } + // We already know that `bucketInfo` is a linked one, + // so we skip one `getBucketInfo` and start with the known link. + ResolvedBucket resolvedBucket = + resolveBucketLink(Pair.of( + bucketInfo.getSourceVolume(), + bucketInfo.getSourceBucket()), + true); + + // If it is a dangling link it means no real bucket exists, + // for example, it could have been deleted, but the links still present. + if (resolvedBucket.isDangling()) { + return bucketInfo; + } + OmBucketInfo realBucket = getResolvedSourceBucket(resolvedBucket, resolvedSourceCache); + return bucketInfo.withOperationalPropertiesFrom(realBucket); + } + + private OmBucketInfo getResolvedSourceBucket( + ResolvedBucket resolvedBucket, + Map<Pair<String, String>, OmBucketInfo> resolvedSourceCache) + throws IOException { + Pair<String, String> sourceKey = Pair.of( + resolvedBucket.realVolume(), + resolvedBucket.realBucket()); + if (resolvedSourceCache != null) { + OmBucketInfo cachedSource = resolvedSourceCache.get(sourceKey); + if (cachedSource != null) { + return cachedSource; + } + } + OmBucketInfo realBucket = bucketManager.getBucketInfo( + resolvedBucket.realVolume(), + resolvedBucket.realBucket()); + if (resolvedSourceCache != null) { + resolvedSourceCache.put(sourceKey, realBucket); + } + return realBucket; + } + + /** * Gets the bucket information. * * @param volume - Volume name. @@ -3042,46 +3114,8 @@ public OmBucketInfo getBucketInfo(String volume, String bucket) } metrics.incNumBucketInfos(); - OmBucketInfo bucketInfo = bucketManager.getBucketInfo(volume, bucket); - - // No links - return the bucket info right away. - if (!bucketInfo.isLink()) { - return bucketInfo; - } - // Otherwise follow the links to find the real bucket. - // We already know that `bucketInfo` is a linked one, - // so we skip one `getBucketInfo` and start with the known link. - ResolvedBucket resolvedBucket = - resolveBucketLink(Pair.of( - bucketInfo.getSourceVolume(), - bucketInfo.getSourceBucket()), - true); - - // If it is a dangling link it means no real bucket exists, - // for example, it could have been deleted, but the links still present. - if (!resolvedBucket.isDangling()) { - OmBucketInfo realBucket = - bucketManager.getBucketInfo( - resolvedBucket.realVolume(), - resolvedBucket.realBucket()); - // Pass the real bucket metadata in the link bucket info. - return bucketInfo.toBuilder() - .setDefaultReplicationConfig( - realBucket.getDefaultReplicationConfig()) - .setIsVersionEnabled(realBucket.getIsVersionEnabled()) - .setStorageType(realBucket.getStorageType()) - .setQuotaInBytes(realBucket.getQuotaInBytes()) - .setQuotaInNamespace(realBucket.getQuotaInNamespace()) - .setUsedBytes(realBucket.getUsedBytes()) - .setSnapshotUsedBytes(realBucket.getSnapshotUsedBytes()) - .setSnapshotUsedNamespace(realBucket.getSnapshotUsedNamespace()) - .setUsedNamespace(realBucket.getUsedNamespace()) - .addAllMetadata(realBucket.getMetadata()) - .setBucketLayout(realBucket.getBucketLayout()) - .build(); - } - // If no real bucket exists, return the requested one's info. - return bucketInfo; + return enrichLinkBucketInfo( + bucketManager.getBucketInfo(volume, bucket)); } catch (Exception ex) { metrics.incNumBucketInfoFails(); auditSuccess = false;
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/DAGResourceLockTracker.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/DAGResourceLockTracker.java index 7fd4405..a669eec 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/DAGResourceLockTracker.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/DAGResourceLockTracker.java
@@ -72,6 +72,11 @@ public static DAGResourceLockTracker get() { return instance; } + @Override + Class<DAGLeveledResource> getResourceClass() { + return DAGLeveledResource.class; + } + /** * Performs a Depth-First Search (DFS) traversal on a directed acyclic graph (DAG) * composed of {@code DAGLeveledResource} objects. This method populates a mapping
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/LeveledResourceLockTracker.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/LeveledResourceLockTracker.java index bbe9cd9..783652a5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/LeveledResourceLockTracker.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/LeveledResourceLockTracker.java
@@ -19,6 +19,7 @@ import java.util.Arrays; import java.util.stream.Stream; +import org.apache.hadoop.ozone.om.lock.OzoneManagerLock.LeveledResource; /** * The LeveledResourceLockTracker class is a singleton that extends the @@ -57,6 +58,11 @@ final class LeveledResourceLockTracker extends ResourceLockTracker<OzoneManagerL private LeveledResourceLockTracker() { } + @Override + Class<LeveledResource> getResourceClass() { + return LeveledResource.class; + } + public static LeveledResourceLockTracker get() { if (instance == null) { synchronized (LeveledResourceLockTracker.class) {
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneManagerLock.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneManagerLock.java index f567f17..b0abd85 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneManagerLock.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneManagerLock.java
@@ -17,14 +17,12 @@ package org.apache.hadoop.ozone.om.lock; -import static org.apache.hadoop.hdds.utils.CompositeKey.combineKeys; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_MANAGER_FAIR_LOCK; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_MANAGER_FAIR_LOCK_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_MANAGER_STRIPED_LOCK_SIZE_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_MANAGER_STRIPED_LOCK_SIZE_PREFIX; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.Striped; import java.util.ArrayList; import java.util.Arrays; @@ -35,19 +33,18 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.StreamSupport; -import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.utils.CompositeKey; import org.apache.hadoop.hdds.utils.SimpleStriped; import org.apache.hadoop.ipc_.ProcessingDetails.Timing; import org.apache.hadoop.ipc_.Server; import org.apache.hadoop.util.Time; +import org.apache.ratis.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -97,41 +94,140 @@ public class OzoneManagerLock implements IOzoneManagerLock { private static final Logger LOG = LoggerFactory.getLogger(OzoneManagerLock.class); - private final Map<Class<? extends Resource>, - Pair<Map<Resource, Striped<ReadWriteLock>>, ResourceLockTracker>> resourcelockMap; + private final ResourceLocks<LeveledResource> leveledResourceLocks; + private final ResourceLocks<DAGLeveledResource> dagLeveledResourceLocks; - private OMLockMetrics omLockMetrics; + private final OMLockMetrics omLockMetrics = OMLockMetrics.create(); + + class ResourceLocks<R extends Resource> { + private final Map<R, Striped<ReentrantReadWriteLock>> lockMap; + private final ResourceLockTracker<R> tracker; + + ResourceLocks(Map<R, Striped<ReentrantReadWriteLock>> lockMap, ResourceLockTracker<R> tracker) { + this.lockMap = lockMap; + this.tracker = tracker; + } + + R assertAcquire(Resource resource) { + final R r = Preconditions.assertInstanceOf(resource, tracker.getResourceClass()); + tracker.clearLockDetails(); + if (!tracker.canLockResource(r)) { + final String errorMessage = "Thread '" + Thread.currentThread().getName() + "' cannot acquire " + + r.getName() + " lock while holding " + getCurrentLocks() + " lock(s)."; + LOG.error(errorMessage); + // TODO: change it to IllegalStateException + throw new RuntimeException(errorMessage); + } + return r; + } + + private ReentrantReadWriteLock getLockForTesting(Resource resource, String... keys) { + final R r = Preconditions.assertInstanceOf(resource, tracker.getResourceClass()); + return getLock(r, keys); + } + + private ReentrantReadWriteLock getLock(R r, String... keys) { + return lockMap.get(r).get(CompositeKey.combineKeys(keys)); + } + + private void acquireLock(R resource, boolean isRead, ReentrantReadWriteLock lock, long startWaitingTimeNanos) { + if (isRead) { + lock.readLock().lock(); + updateReadLockMetrics(resource, tracker, lock, startWaitingTimeNanos); + } else { + lock.writeLock().lock(); + updateWriteLockMetrics(resource, tracker, lock, startWaitingTimeNanos); + } + } + + OMLockDetails acquire(Resource resource, boolean isRead, + Function<Striped<ReentrantReadWriteLock>, Iterable<ReentrantReadWriteLock>> getLocks) { + final R r = assertAcquire(resource); + final long startWaitingTimeNanos = Time.monotonicNowNanos(); + for (ReentrantReadWriteLock lock : getLocks.apply(lockMap.get(r))) { + acquireLock(r, isRead, lock, startWaitingTimeNanos); + } + return tracker.lockResource(r); + } + + OMLockDetails acquire(Resource resource, boolean isRead, String... keys) { + final R r = assertAcquire(resource); + final long startWaitingTimeNanos = Time.monotonicNowNanos(); + acquireLock(r, isRead, getLock(r, keys), startWaitingTimeNanos); + return tracker.lockResource(r); + } + + void releaseLock(R resource, boolean isRead, ReentrantReadWriteLock lock) { + if (isRead) { + lock.readLock().unlock(); + updateReadUnlockMetrics(resource, tracker, lock); + } else { + boolean isWriteLocked = lock.isWriteLockedByCurrentThread(); + lock.writeLock().unlock(); + updateWriteUnlockMetrics(resource, tracker, lock, isWriteLocked); + } + } + + OMLockDetails release(Resource resource, boolean isRead, String... keys) { + final R r = Preconditions.assertInstanceOf(resource, tracker.getResourceClass()); + tracker.clearLockDetails(); + final ReentrantReadWriteLock lock = getLock(r, keys); + releaseLock(r, isRead, lock); + return tracker.unlockResource(r); + } + + private OMLockDetails release(Resource resource, boolean isRead, + Function<Striped<ReentrantReadWriteLock>, Iterable<ReentrantReadWriteLock>> getLock) { + final R r = Preconditions.assertInstanceOf(resource, tracker.getResourceClass()); + tracker.clearLockDetails(); + final Iterable<ReentrantReadWriteLock> i = getLock.apply(lockMap.get(r)); + final List<ReentrantReadWriteLock> locks = StreamSupport.stream(i.spliterator(), false) + .collect(Collectors.toList()); + // Release locks in reverse order. + Collections.reverse(locks); + for (ReentrantReadWriteLock lock : locks) { + releaseLock(r, isRead, lock); + } + return tracker.unlockResource(r); + } + + List<String> getCurrentLocks() { + return tracker.getCurrentLockedResources() + .map(Resource::getName) + .collect(Collectors.toList()); + } + } /** * Creates new OzoneManagerLock instance. * @param conf Configuration object */ public OzoneManagerLock(ConfigurationSource conf) { - omLockMetrics = OMLockMetrics.create(); - this.resourcelockMap = ImmutableMap.of(LeveledResource.class, getLeveledLocks(conf), DAGLeveledResource.class, - getFlatLocks(conf)); + this.leveledResourceLocks = newResourceLocks(LeveledResourceLockTracker.get(), conf); + this.dagLeveledResourceLocks = newResourceLocks(DAGResourceLockTracker.get(), conf); } - private Pair<Map<Resource, Striped<ReadWriteLock>>, ResourceLockTracker> getLeveledLocks( - ConfigurationSource conf) { - Map<LeveledResource, Striped<ReadWriteLock>> stripedLockMap = new EnumMap<>(LeveledResource.class); - for (LeveledResource r : LeveledResource.values()) { + private <T extends Enum<T> & Resource> ResourceLocks<T> newResourceLocks( + ResourceLockTracker<T> tracker, ConfigurationSource conf) { + final Class<T> clazz = tracker.getResourceClass(); + final EnumMap<T, Striped<ReentrantReadWriteLock>> stripedLockMap = new EnumMap<>(clazz); + for (T r : clazz.getEnumConstants()) { stripedLockMap.put(r, createStripeLock(r, conf)); } - return Pair.of(Collections.unmodifiableMap(stripedLockMap), LeveledResourceLockTracker.get()); + return new ResourceLocks<>(Collections.unmodifiableMap(stripedLockMap), tracker); } - private Pair<Map<Resource, Striped<ReadWriteLock>>, ResourceLockTracker> getFlatLocks( - ConfigurationSource conf) { - Map<DAGLeveledResource, Striped<ReadWriteLock>> stripedLockMap = new EnumMap<>(DAGLeveledResource.class); - for (DAGLeveledResource r : DAGLeveledResource.values()) { - stripedLockMap.put(r, createStripeLock(r, conf)); + private ResourceLocks<?> getResourceLocks(Resource instance) { + final Class<?> clazz = instance.getClass(); + if (clazz == LeveledResource.class) { + return leveledResourceLocks; + } else if (clazz == DAGLeveledResource.class) { + return dagLeveledResourceLocks; } - return Pair.of(Collections.unmodifiableMap(stripedLockMap), DAGResourceLockTracker.get()); + throw new IllegalArgumentException("Unsupported resource class: " + clazz); } - private Striped<ReadWriteLock> createStripeLock(Resource r, - ConfigurationSource conf) { + private static Striped<ReentrantReadWriteLock> createStripeLock(Resource r, ConfigurationSource conf) { boolean fair = conf.getBoolean(OZONE_MANAGER_FAIR_LOCK, OZONE_MANAGER_FAIR_LOCK_DEFAULT); String stripeSizeKey = OZONE_MANAGER_STRIPED_LOCK_SIZE_PREFIX + @@ -141,11 +237,12 @@ private Striped<ReadWriteLock> createStripeLock(Resource r, return SimpleStriped.readWriteLock(size, fair); } - private Iterable<ReadWriteLock> getAllLocks(Striped<ReadWriteLock> striped) { + private Iterable<ReentrantReadWriteLock> getAllLocks(Striped<ReentrantReadWriteLock> striped) { return IntStream.range(0, striped.size()).mapToObj(striped::getAt).collect(Collectors.toList()); } - private Iterable<ReadWriteLock> bulkGetLock(Striped<ReadWriteLock> striped, Collection<String[]> keys) { + private Iterable<ReentrantReadWriteLock> bulkGetLock(Striped<ReentrantReadWriteLock> striped, + Collection<String[]> keys) { List<Object> lockKeys = new ArrayList<>(keys.size()); for (String[] key : keys) { if (Objects.nonNull(key)) { @@ -155,13 +252,6 @@ private Iterable<ReadWriteLock> bulkGetLock(Striped<ReadWriteLock> striped, Coll return striped.bulkGet(lockKeys); } - private ReentrantReadWriteLock getLock(Map<Resource, Striped<ReadWriteLock>> lockMap, Resource resource, - String... keys) { - Striped<ReadWriteLock> striped = lockMap.get(resource); - Object key = combineKeys(keys); - return (ReentrantReadWriteLock) striped.get(key); - } - /** * Acquire read lock on resource. * @@ -181,7 +271,8 @@ private ReentrantReadWriteLock getLock(Map<Resource, Striped<ReadWriteLock>> loc */ @Override public OMLockDetails acquireReadLock(Resource resource, String... keys) { - return acquireLock(resource, true, keys); + return getResourceLocks(resource) + .acquire(resource, true, keys); } /** @@ -203,7 +294,8 @@ public OMLockDetails acquireReadLock(Resource resource, String... keys) { */ @Override public OMLockDetails acquireReadLocks(Resource resource, Collection<String[]> keys) { - return acquireLocks(resource, true, striped -> bulkGetLock(striped, keys)); + return getResourceLocks(resource) + .acquire(resource, true, striped -> bulkGetLock(striped, keys)); } /** @@ -225,7 +317,8 @@ public OMLockDetails acquireReadLocks(Resource resource, Collection<String[]> ke */ @Override public OMLockDetails acquireWriteLock(Resource resource, String... keys) { - return acquireLock(resource, false, keys); + return getResourceLocks(resource) + .acquire(resource, false, keys); } /** @@ -247,7 +340,8 @@ public OMLockDetails acquireWriteLock(Resource resource, String... keys) { */ @Override public OMLockDetails acquireWriteLocks(Resource resource, Collection<String[]> keys) { - return acquireLocks(resource, false, striped -> bulkGetLock(striped, keys)); + return getResourceLocks(resource) + .acquire(resource, false, striped -> bulkGetLock(striped, keys)); } /** @@ -257,59 +351,11 @@ public OMLockDetails acquireWriteLocks(Resource resource, Collection<String[]> k */ @Override public OMLockDetails acquireResourceWriteLock(Resource resource) { - return acquireLocks(resource, false, this::getAllLocks); + return getResourceLocks(resource) + .acquire(resource, false, this::getAllLocks); } - private void acquireLock(Resource resource, boolean isReadLock, ReadWriteLock lock, - long startWaitingTimeNanos) { - if (isReadLock) { - lock.readLock().lock(); - updateReadLockMetrics(resource, (ReentrantReadWriteLock) lock, startWaitingTimeNanos); - } else { - lock.writeLock().lock(); - updateWriteLockMetrics(resource, (ReentrantReadWriteLock) lock, startWaitingTimeNanos); - } - } - - private OMLockDetails acquireLocks(Resource resource, boolean isReadLock, - Function<Striped<ReadWriteLock>, Iterable<ReadWriteLock>> lockListProvider) { - Pair<Map<Resource, Striped<ReadWriteLock>>, ResourceLockTracker> resourceLockPair = - resourcelockMap.get(resource.getClass()); - ResourceLockTracker<Resource> resourceLockTracker = resourceLockPair.getRight(); - resourceLockTracker.clearLockDetails(); - if (!resourceLockTracker.canLockResource(resource)) { - String errorMessage = getErrorMessage(resource); - LOG.error(errorMessage); - throw new RuntimeException(errorMessage); - } - - long startWaitingTimeNanos = Time.monotonicNowNanos(); - - for (ReadWriteLock lock : lockListProvider.apply(resourceLockPair.getKey().get(resource))) { - acquireLock(resource, isReadLock, lock, startWaitingTimeNanos); - } - return resourceLockTracker.lockResource(resource); - } - - private OMLockDetails acquireLock(Resource resource, boolean isReadLock, String... keys) { - Pair<Map<Resource, Striped<ReadWriteLock>>, ResourceLockTracker> resourceLockPair = - resourcelockMap.get(resource.getClass()); - ResourceLockTracker<Resource> resourceLockTracker = resourceLockPair.getRight(); - resourceLockTracker.clearLockDetails(); - if (!resourceLockTracker.canLockResource(resource)) { - String errorMessage = getErrorMessage(resource); - LOG.error(errorMessage); - throw new RuntimeException(errorMessage); - } - - long startWaitingTimeNanos = Time.monotonicNowNanos(); - - ReentrantReadWriteLock lock = getLock(resourceLockPair.getKey(), resource, keys); - acquireLock(resource, isReadLock, lock, startWaitingTimeNanos); - return resourceLockTracker.lockResource(resource); - } - - private void updateReadLockMetrics(Resource resource, + private void updateReadLockMetrics(Resource resource, ResourceLockTracker<? extends Resource> tracker, ReentrantReadWriteLock lock, long startWaitingTimeNanos) { /* @@ -323,14 +369,13 @@ private void updateReadLockMetrics(Resource resource, // Adds a snapshot to the metric readLockWaitingTimeMsStat. omLockMetrics.setReadLockWaitingTimeMsStat( TimeUnit.NANOSECONDS.toMillis(readLockWaitingTimeNanos)); - updateProcessingDetails(resourcelockMap.get(resource.getClass()).getValue(), - Timing.LOCKWAIT, readLockWaitingTimeNanos); + updateProcessingDetails(tracker, Timing.LOCKWAIT, readLockWaitingTimeNanos); resource.getResourceManager().setStartReadHeldTimeNanos(Time.monotonicNowNanos()); } } - private void updateWriteLockMetrics(Resource resource, + private void updateWriteLockMetrics(Resource resource, ResourceLockTracker<? extends Resource> tracker, ReentrantReadWriteLock lock, long startWaitingTimeNanos) { /* * writeHoldCount helps in metrics updation only once in case @@ -345,25 +390,15 @@ private void updateWriteLockMetrics(Resource resource, // Adds a snapshot to the metric writeLockWaitingTimeMsStat. omLockMetrics.setWriteLockWaitingTimeMsStat( TimeUnit.NANOSECONDS.toMillis(writeLockWaitingTimeNanos)); - updateProcessingDetails(resourcelockMap.get(resource.getClass()).getValue(), Timing.LOCKWAIT, - writeLockWaitingTimeNanos); + updateProcessingDetails(tracker, Timing.LOCKWAIT, writeLockWaitingTimeNanos); resource.getResourceManager().setStartWriteHeldTimeNanos(Time.monotonicNowNanos()); } } - private String getErrorMessage(Resource resource) { - return "Thread '" + Thread.currentThread().getName() + "' cannot " + - "acquire " + resource.getName() + " lock while holding " + - getCurrentLocks().toString() + " lock(s)."; - } - @VisibleForTesting - List<String> getCurrentLocks() { - return resourcelockMap.values().stream().map(Pair::getValue) - .flatMap(rlm -> ((ResourceLockTracker<? extends Resource>)rlm).getCurrentLockedResources()) - .map(Resource::getName) - .collect(Collectors.toList()); + int getCurrentLockSizeForTesting() { + return leveledResourceLocks.getCurrentLocks().size() + dagLeveledResourceLocks.getCurrentLocks().size(); } /** @@ -397,7 +432,8 @@ public void releaseMultiUserLock(String firstUser, String secondUser) { */ @Override public OMLockDetails releaseWriteLock(Resource resource, String... keys) { - return releaseLock(resource, false, keys); + return getResourceLocks(resource) + .release(resource, false, keys); } /** @@ -410,7 +446,8 @@ public OMLockDetails releaseWriteLock(Resource resource, String... keys) { */ @Override public OMLockDetails releaseWriteLocks(Resource resource, Collection<String[]> keys) { - return releaseLocks(resource, false, striped -> bulkGetLock(striped, keys)); + return getResourceLocks(resource) + .release(resource, false, striped -> bulkGetLock(striped, keys)); } /** @@ -420,7 +457,8 @@ public OMLockDetails releaseWriteLocks(Resource resource, Collection<String[]> k */ @Override public OMLockDetails releaseResourceWriteLock(Resource resource) { - return releaseLocks(resource, false, this::getAllLocks); + return getResourceLocks(resource) + .release(resource, false, this::getAllLocks); } /** @@ -433,7 +471,8 @@ public OMLockDetails releaseResourceWriteLock(Resource resource) { */ @Override public OMLockDetails releaseReadLock(Resource resource, String... keys) { - return releaseLock(resource, true, keys); + return getResourceLocks(resource) + .release(resource, true, keys); } /** @@ -446,51 +485,11 @@ public OMLockDetails releaseReadLock(Resource resource, String... keys) { */ @Override public OMLockDetails releaseReadLocks(Resource resource, Collection<String[]> keys) { - return releaseLocks(resource, true, striped -> bulkGetLock(striped, keys)); + return getResourceLocks(resource) + .release(resource, true, striped -> bulkGetLock(striped, keys)); } - private OMLockDetails releaseLock(Resource resource, boolean isReadLock, - String... keys) { - Pair<Map<Resource, Striped<ReadWriteLock>>, ResourceLockTracker> resourceLockPair = - resourcelockMap.get(resource.getClass()); - ResourceLockTracker<Resource> resourceLockTracker = resourceLockPair.getRight(); - resourceLockTracker.clearLockDetails(); - ReentrantReadWriteLock lock = getLock(resourceLockPair.getKey(), resource, keys); - if (isReadLock) { - lock.readLock().unlock(); - updateReadUnlockMetrics(resource, lock); - } else { - boolean isWriteLocked = lock.isWriteLockedByCurrentThread(); - lock.writeLock().unlock(); - updateWriteUnlockMetrics(resource, lock, isWriteLocked); - } - return resourceLockTracker.unlockResource(resource); - } - - private OMLockDetails releaseLocks(Resource resource, boolean isReadLock, - Function<Striped<ReadWriteLock>, Iterable<ReadWriteLock>> lockListProvider) { - Pair<Map<Resource, Striped<ReadWriteLock>>, ResourceLockTracker> resourceLockPair = - resourcelockMap.get(resource.getClass()); - ResourceLockTracker<Resource> resourceLockTracker = resourceLockPair.getRight(); - resourceLockTracker.clearLockDetails(); - List<ReadWriteLock> locks = StreamSupport.stream(lockListProvider.apply(resourceLockPair.getKey().get(resource)) - .spliterator(), false).collect(Collectors.toList()); - // Release locks in reverse order. - Collections.reverse(locks); - for (ReadWriteLock lock : locks) { - if (isReadLock) { - lock.readLock().unlock(); - updateReadUnlockMetrics(resource, (ReentrantReadWriteLock) lock); - } else { - boolean isWriteLocked = ((ReentrantReadWriteLock)lock).isWriteLockedByCurrentThread(); - lock.writeLock().unlock(); - updateWriteUnlockMetrics(resource, (ReentrantReadWriteLock) lock, isWriteLocked); - } - } - return resourceLockTracker.unlockResource(resource); - } - - private void updateReadUnlockMetrics(Resource resource, + private void updateReadUnlockMetrics(Resource resource, ResourceLockTracker<? extends Resource> tracker, ReentrantReadWriteLock lock) { /* * readHoldCount helps in metrics updation only once in case @@ -503,12 +502,11 @@ private void updateReadUnlockMetrics(Resource resource, // Adds a snapshot to the metric readLockHeldTimeMsStat. omLockMetrics.setReadLockHeldTimeMsStat( TimeUnit.NANOSECONDS.toMillis(readLockHeldTimeNanos)); - updateProcessingDetails(resourcelockMap.get(resource.getClass()).getValue(), Timing.LOCKSHARED, - readLockHeldTimeNanos); + updateProcessingDetails(tracker, Timing.LOCKSHARED, readLockHeldTimeNanos); } } - private void updateWriteUnlockMetrics(Resource resource, + private void updateWriteUnlockMetrics(Resource resource, ResourceLockTracker<? extends Resource> tracker, ReentrantReadWriteLock lock, boolean isWriteLocked) { /* * writeHoldCount helps in metrics updation only once in case @@ -522,8 +520,7 @@ private void updateWriteUnlockMetrics(Resource resource, // Adds a snapshot to the metric writeLockHeldTimeMsStat. omLockMetrics.setWriteLockHeldTimeMsStat( TimeUnit.NANOSECONDS.toMillis(writeLockHeldTimeNanos)); - updateProcessingDetails(resourcelockMap.get(resource.getClass()).getValue(), Timing.LOCKEXCLUSIVE, - writeLockHeldTimeNanos); + updateProcessingDetails(tracker, Timing.LOCKEXCLUSIVE, writeLockHeldTimeNanos); } } @@ -535,7 +532,7 @@ private void updateWriteUnlockMetrics(Resource resource, @Override @VisibleForTesting public int getReadHoldCount(Resource resource, String... keys) { - return getLock(resourcelockMap.get(resource.getClass()).getKey(), resource, keys).getReadHoldCount(); + return getResourceLocks(resource).getLockForTesting(resource, keys).getReadHoldCount(); } @@ -547,7 +544,7 @@ public int getReadHoldCount(Resource resource, String... keys) { @Override @VisibleForTesting public int getWriteHoldCount(Resource resource, String... keys) { - return getLock(resourcelockMap.get(resource.getClass()).getKey(), resource, keys).getWriteHoldCount(); + return getResourceLocks(resource).getLockForTesting(resource, keys).getWriteHoldCount(); } /** @@ -559,9 +556,8 @@ public int getWriteHoldCount(Resource resource, String... keys) { */ @Override @VisibleForTesting - public boolean isWriteLockedByCurrentThread(Resource resource, - String... keys) { - return getLock(resourcelockMap.get(resource.getClass()).getKey(), resource, keys).isWriteLockedByCurrentThread(); + public boolean isWriteLockedByCurrentThread(Resource resource, String... keys) { + return getResourceLocks(resource).getLockForTesting(resource, keys).isWriteLockedByCurrentThread(); } /**
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/ResourceLockTracker.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/ResourceLockTracker.java index 8a551e5..80e4071 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/ResourceLockTracker.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/ResourceLockTracker.java
@@ -31,6 +31,8 @@ abstract class ResourceLockTracker<T extends IOzoneManagerLock.Resource> { private final ThreadLocal<OMLockDetails> omLockDetails = ThreadLocal.withInitial(OMLockDetails::new); + abstract Class<T> getResourceClass(); + abstract boolean canLockResource(T resource); abstract Stream<T> getCurrentLockedResources();
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java index d12b8fa..4e22533 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
@@ -57,11 +57,14 @@ import org.apache.hadoop.hdds.client.ContainerBlockID; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto; import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock; import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; import org.apache.hadoop.hdds.scm.exceptions.SCMException; +import org.apache.hadoop.hdds.scm.net.NetworkTopology; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; @@ -69,6 +72,7 @@ import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.KeyManager; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmConfig; import org.apache.hadoop.ozone.om.OzoneManager; @@ -189,15 +193,38 @@ protected List<OmKeyLocationInfo> allocateBlock( UserInfo userInfo, OzoneManager ozoneManager) throws IOException { final long scmBlockSize = ozoneManager.getScmBlockSize(); + final KeyManager keyManager = ozoneManager.getKeyManager(); int dataGroupSize = replicationConfig instanceof ECReplicationConfig ? ((ECReplicationConfig) replicationConfig).getData() : 1; final int numBlocks = (int) Math.min(ozoneManager.getPreallocateBlocksMax(), (requestedSize - 1) / (scmBlockSize * dataGroupSize) + 1); - String clientMachine = ""; - if (shouldSortDatanodes) { - clientMachine = userInfo.getRemoteAddress(); + final String scmClientMachine; + final String omClientMachine; + // Sorted order cached by datanode set so blocks whose pipelines share the + // same datanodes are sorted once (mirrors the read path's caching). Keyed by + // the UUID set so it is order-insensitive and dedups across pipelines. + final Map<Set<String>, List<? extends DatanodeDetails>> sortedByNodes; + final String remoteAddress = userInfo.getRemoteAddress(); + final NetworkTopology clusterMap = shouldSortDatanodes + && keyManager.isSortDatanodesForWriteEnabled() + ? ozoneManager.getClusterMapAllowNull() : null; + if (!shouldSortDatanodes) { + scmClientMachine = ""; + omClientMachine = ""; + sortedByNodes = null; + } else if (clusterMap != null && !remoteAddress.isEmpty()) { + // Sort in OM: SCM skips sorting (empty machine), OM sorts by remoteAddress. + scmClientMachine = ""; + omClientMachine = remoteAddress; + sortedByNodes = new HashMap<>(); + } else { + // Sort in SCM (or keep order when remoteAddress is empty, since SCM skips + // sorting for an empty client machine). + scmClientMachine = remoteAddress; + omClientMachine = ""; + sortedByNodes = null; } List<OmKeyLocationInfo> locationInfos = new ArrayList<>(numBlocks); @@ -205,7 +232,7 @@ protected List<OmKeyLocationInfo> allocateBlock( final List<AllocatedBlock> allocatedBlocks; try { allocatedBlocks = ozoneManager.getScmClient().getBlockClient().allocateBlock( - scmBlockSize, numBlocks, replicationConfig, ozoneManager.getOMServiceId(), excludeList, clientMachine); + scmBlockSize, numBlocks, replicationConfig, ozoneManager.getOMServiceId(), excludeList, scmClientMachine); } catch (SCMException ex) { ozoneManager.getMetrics().incNumBlockAllocateCallFails(); if (ex.getResult() == SCMException.ResultCodes.SAFE_MODE_EXCEPTION) { @@ -216,11 +243,30 @@ protected List<OmKeyLocationInfo> allocateBlock( } for (AllocatedBlock allocatedBlock : allocatedBlocks) { BlockID blockID = new BlockID(allocatedBlock.getBlockID()); + Pipeline pipeline = allocatedBlock.getPipeline(); + if (sortedByNodes != null) { + final List<DatanodeDetails> nodes = pipeline.getNodes(); + final Set<String> uuidSet = nodes.stream() + .map(DatanodeDetails::getUuidString).collect(Collectors.toSet()); + List<? extends DatanodeDetails> sorted = sortedByNodes.get(uuidSet); + if (sorted == null) { + sorted = keyManager.sortDatanodesForWrite(nodes, omClientMachine, clusterMap); + // Cache only a freshly sorted order, not an input list returned + // unchanged when the client is unresolved: that order is per-pipeline + // and must not be reused for another pipeline with the same node set. + if (sorted != nodes) { + sortedByNodes.put(uuidSet, sorted); + } + } + if (!Objects.equals(sorted, pipeline.getNodesInOrder())) { + pipeline = pipeline.copyWithNodesInOrder(sorted); + } + } OmKeyLocationInfo.Builder builder = new OmKeyLocationInfo.Builder() .setBlockID(blockID) .setLength(scmBlockSize) .setOffset(0) - .setPipeline(allocatedBlock.getPipeline()); + .setPipeline(pipeline); if (ozoneManager.isGrpcBlockTokenEnabled()) { final Token<OzoneBlockTokenIdentifier> token = ozoneManager.getBlockTokenSecretManager().generateToken( remoteUser, blockID, READ_WRITE, scmBlockSize);
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java index 22f470c..72637c1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java
@@ -100,10 +100,14 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { KeyArgs resolvedArgs = resolveBucketAndCheckKeyAcls(newKeyArgs.build(), ozoneManager, ACLType.CREATE); + int schemaVersion = resolveMultipartSchemaVersion(ozoneManager); + MultipartInfoInitiateRequest.Builder requestBuilder = + multipartInfoInitiateRequest.toBuilder() + .setKeyArgs(resolvedArgs) + .setSchemaVersion(schemaVersion); return getOmRequest().toBuilder() .setUserInfo(getUserInfo()) - .setInitiateMultiPartUploadRequest( - multipartInfoInitiateRequest.toBuilder().setKeyArgs(resolvedArgs)) + .setInitiateMultiPartUploadRequest(requestBuilder) .build(); } @@ -195,6 +199,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut replicationConfig) .setObjectID(objectID) .setUpdateID(transactionLogIndex) + // Source of truth is the value stamped onto the proto in preExecute + // (before Ratis). Never re-check MLV here in the replicated apply path. + .setSchemaVersion(multipartInfoInitiateRequest.getSchemaVersion()) .build(); omKeyInfo = new OmKeyInfo.Builder() @@ -284,6 +291,34 @@ protected void logResult(OzoneManager ozoneManager, } } + /** + * Resolve the schema version stamped onto a newly initiated multipart upload. + * <p> + * This is a server-authoritative decision made once, in {@code preExecute} + * (i.e. on the leader, before the request is submitted to Ratis). Any + * client-supplied {@code schemaVersion} on the request is intentionally + * ignored so that a client can never force the OM to persist an on-disk + * format the cluster is not ready for. + * <p> + * The split parts-table on-disk format is gated on the + * {@link OMLayoutFeature#MPU_PARTS_TABLE_SPLIT} layout feature: + * <ul> + * <li>pre-finalized (or mixed-binary rolling upgrade) → legacy schema, + * so no split-table rows are written on a cluster that may still be + * downgraded;</li> + * <li>finalized → split parts-table schema.</li> + * </ul> + * Because finalization is replicated through Ratis, the leader's view here is + * consistent across the quorum, and the stamped value (not the live layout + * version) is what all subsequent processing obeys. + */ + protected int resolveMultipartSchemaVersion(OzoneManager ozoneManager) { + return ozoneManager.getVersionManager() + .isAllowed(OMLayoutFeature.MPU_PARTS_TABLE_SPLIT) + ? OmMultipartKeyInfo.SPLIT_PARTS_TABLE_SCHEMA_VERSION + : OmMultipartKeyInfo.LEGACY_SCHEMA_VERSION; + } + @RequestFeatureValidator( conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, processingPhase = RequestProcessingPhase.PRE_PROCESS,
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequestWithFSO.java index 919491d..5596be4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequestWithFSO.java
@@ -168,6 +168,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut .setObjectID(pathInfoFSO.getLeafNodeObjectId()) .setUpdateID(transactionLogIndex) .setParentID(pathInfoFSO.getLastKnownParentId()) + // Source of truth is the value stamped onto the proto in preExecute + // (before Ratis). Never re-check MLV here in the replicated apply path. + .setSchemaVersion(multipartInfoInitiateRequest.getSchemaVersion()) .build(); omKeyInfo = new OmKeyInfo.Builder()
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java index 78f1af9..24fe698 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java
@@ -223,7 +223,15 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut // Add this part information in to multipartKeyInfo. multipartKeyInfo.addPartKeyInfo(partKeyInfo.build()); } else { - validateSplitPartInfo(omKeyInfo, partNumber); + // an ETag is MANDATORY for every committed part in the split parts-table schema, + // enforced server-side for ALL clients (S3 gateway and native Ozone client alike). + // The S3 gateway computes the MD5 ETag on upload; any other client must also supply one. + // Reject the commit early with a clear INVALID_REQUEST if it is missing, + if (StringUtils.isBlank(omKeyInfo.getMetadata().get(OzoneConsts.ETAG))) { + throw new OMException( + "Missing ETag for multipart upload part " + partNumber, + OMException.ResultCodes.INVALID_REQUEST); + } multipartPartInfo = OmMultipartPartInfo.from(partName, partNumber, omKeyInfo); omMetadataManager.getMultipartPartsTable().addCacheEntry( new CacheKey<>(multipartPartKey), @@ -410,14 +418,6 @@ private String getMultipartKey(String volumeName, String bucketName, keyName, uploadID); } - private void validateSplitPartInfo(OmKeyInfo omKeyInfo, int partNumber) - throws OMException { - if (StringUtils.isBlank(omKeyInfo.getMetadata().get(OzoneConsts.ETAG))) { - throw new OMException("Missing ETag for multipart upload part " - + partNumber, OMException.ResultCodes.INVALID_REQUEST); - } - } - @RequestFeatureValidator( conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, processingPhase = RequestProcessingPhase.PRE_PROCESS,
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotDeleteRequest.java index 03f3aa0..32938ae 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotDeleteRequest.java
@@ -25,6 +25,7 @@ import java.io.IOException; import java.nio.file.InvalidPathException; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.OmUtils; @@ -181,6 +182,13 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut snapshotInfo.setSnapshotStatus( SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED); snapshotInfo.setDeletionTime(deletionTime); + // Stamp the deletion transaction so that areSnapshotChangesFlushedToDB() reports the deletion as + // unflushed until the double buffer persists it. SnapshotDeletingService relies on this + // (shouldIgnoreSnapshot) to defer moveTableKeys/purge; without the stamp the purge can be applied and + // empty the in-memory snapshot chain while the on-disk snapshotInfoTable still holds the snapshot as + // ACTIVE, letting KeyDeletingService reclaim blocks that the on-disk snapshot still references. + snapshotInfo.setLastTransactionInfo( + TransactionInfo.valueOf(context.getTermIndex()).toByteString()); // Update table cache first omMetadataManager.getSnapshotInfoTable().addCacheEntry(
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/util/OMMultipartUploadUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/util/OMMultipartUploadUtils.java index 598f7dc..d6fd32a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/util/OMMultipartUploadUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/util/OMMultipartUploadUtils.java
@@ -165,6 +165,16 @@ public static SortedMap<Integer, OmMultipartPartInfo> scanParts( return parts; } + /** + * Count the multipart parts belonging to a given upload in the split + * multipartPartsTable, honouring cache tombstones and pending commits. The + * count therefore matches the set of parts a subsequent abort/cleanup would + * process, which makes it suitable for batch sizing. + */ + public static int countParts(OMMetadataManager omMetadataManager, String uploadId) throws IOException { + return scanParts(omMetadataManager, uploadId).size(); + } + public static List<OmMultipartPartKey> getPartKeys(String uploadId, SortedMap<Integer, OmMultipartPartInfo> parts) { List<OmMultipartPartKey> partKeys = new ArrayList<>(parts.size());
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java index ead8acc..5310f5b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
@@ -1138,11 +1138,23 @@ private void processMultipartUploads(OmBucketInfo bucketInfo, List<OmLCRule> rul abortExpiredMultipartUploadsAndClear(bucketInfo, expiredUploads); } - // Get part count for this MPU (at least 1 even if no parts uploaded yet) - int partCount = Math.max(1, mpuKeyInfo.getPartKeyInfoMap().size()); - expiredUploads.add(upload, partCount); + // Split-schema MPUs keep parts in multipartPartsTable (the embedded map + // is empty); legacy MPUs use the embedded map. An MPU with no uploaded + // parts is valid (S3 allows aborting it with an empty parts list). + int uploadedParts; + try { + uploadedParts = mpuKeyInfo.getSchemaVersion() + == OmMultipartKeyInfo.SPLIT_PARTS_TABLE_SCHEMA_VERSION + ? OMMultipartUploadUtils.countParts(omMetadataManager, upload.getUploadId()) + : mpuKeyInfo.getPartKeyInfoMap().size(); + } catch (IOException e) { + LOG.warn("Failed to count parts for MPU {}/{}/{} uploadId {}, skipping", + volumeName, bucketName, keyName, upload.getUploadId(), e); + break; + } + expiredUploads.add(upload, uploadedParts); LOG.debug("Multipart upload {}/{}/{} with uploadId {} ({} parts) will be aborted", - volumeName, bucketName, keyName, upload.getUploadId(), partCount); + volumeName, bucketName, keyName, upload.getUploadId(), uploadedParts); break; } }
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java index 790ab58..189bb3f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
@@ -45,7 +45,8 @@ public enum OMLayoutFeature implements LayoutFeature { HBASE_SUPPORT(7, "Full support of hsync, lease recovery and listOpenFiles APIs for HBase"), DELEGATION_TOKEN_SYMMETRIC_SIGN(8, "Delegation token signed by symmetric key"), SNAPSHOT_DEFRAG(9, "Supporting defragmentation of snapshot"), - S3_LIFECYCLE_SUPPORT(10, "S3 bucket lifecycle configuration support"); + S3_LIFECYCLE_SUPPORT(10, "S3 bucket lifecycle configuration support"), + MPU_PARTS_TABLE_SPLIT(11, "Split multipart table into separate table for parts and key"); /////////////////////////////// /////////////////////////////
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketManagerImpl.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketManagerImpl.java index 42d7486..1755fb4 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketManagerImpl.java
@@ -31,10 +31,12 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.List; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ECReplicationConfig; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.StorageType; @@ -460,4 +462,82 @@ public void testLinkedBucketResolution() throws Exception { bucketInfo.getIsVersionEnabled(), storedLinkBucket.getIsVersionEnabled()); } + + @Test + public void testListBucketsResolvesFsoAndObsLinkLayouts() throws Exception { + String volume = volumeName(); + createSampleVol(volume); + + ECReplicationConfig fsoReplication = new ECReplicationConfig(3, 2); + OmBucketInfo fsoSource = OmBucketInfo.newBuilder() + .setVolumeName(volume) + .setBucketName("fso-source") + .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED) + .setDefaultReplicationConfig(new DefaultReplicationConfig(fsoReplication)) + .build(); + writeClient.createBucket(fsoSource); + + RatisReplicationConfig obsReplication = + RatisReplicationConfig.getInstance(ReplicationFactor.THREE); + OmBucketInfo obsSource = OmBucketInfo.newBuilder() + .setVolumeName(volume) + .setBucketName("obs-source") + .setBucketLayout(BucketLayout.OBJECT_STORE) + .setDefaultReplicationConfig(new DefaultReplicationConfig(obsReplication)) + .build(); + writeClient.createBucket(obsSource); + + OmBucketInfo fsoLink = OmBucketInfo.newBuilder() + .setVolumeName(volume) + .setBucketName("link-fso") + .setSourceVolume(volume) + .setSourceBucket("fso-source") + .build(); + writeClient.createBucket(fsoLink); + + OmBucketInfo obsLink = OmBucketInfo.newBuilder() + .setVolumeName(volume) + .setBucketName("link-obs") + .setSourceVolume(volume) + .setSourceBucket("obs-source") + .build(); + writeClient.createBucket(obsLink); + + OmBucketInfo fsoLinkInfo = writeClient.getBucketInfo(volume, "link-fso"); + assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, fsoLinkInfo.getBucketLayout()); + assertEquals(fsoReplication, + fsoLinkInfo.getDefaultReplicationConfig().getReplicationConfig()); + + OmBucketInfo obsLinkInfo = writeClient.getBucketInfo(volume, "link-obs"); + assertEquals(BucketLayout.OBJECT_STORE, obsLinkInfo.getBucketLayout()); + assertEquals(obsReplication, + obsLinkInfo.getDefaultReplicationConfig().getReplicationConfig()); + + List<OmBucketInfo> listedBuckets = + writeClient.listBuckets(volume, "", "", 100, false); + + OmBucketInfo listedFsoLink = null; + OmBucketInfo listedObsLink = null; + for (OmBucketInfo listedBucket : listedBuckets) { + if ("link-fso".equals(listedBucket.getBucketName())) { + listedFsoLink = listedBucket; + } else if ("link-obs".equals(listedBucket.getBucketName())) { + listedObsLink = listedBucket; + } + } + assertNotNull(listedFsoLink, "link-fso not found in listBuckets response"); + assertNotNull(listedObsLink, "link-obs not found in listBuckets response"); + + assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, listedFsoLink.getBucketLayout()); + assertEquals(fsoReplication, + listedFsoLink.getDefaultReplicationConfig().getReplicationConfig()); + assertEquals(volume, listedFsoLink.getSourceVolume()); + assertEquals("fso-source", listedFsoLink.getSourceBucket()); + + assertEquals(BucketLayout.OBJECT_STORE, listedObsLink.getBucketLayout()); + assertEquals(obsReplication, + listedObsLink.getDefaultReplicationConfig().getReplicationConfig()); + assertEquals(volume, listedObsLink.getSourceVolume()); + assertEquals("obs-source", listedObsLink.getSourceBucket()); + } }
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java index 9b18442..be307c0 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java
@@ -159,8 +159,7 @@ public void listMultipartUploadPartsWithZeroUpload() throws IOException { } @Test - public void listMultipartUploadPartsWithoutEtagField() throws IOException { - // For backward compatibility reasons + public void listMultipartUploadPartsWithEtagField() throws IOException { final String volume = volumeName(); final String bucket = "bucketForEtag"; final String key = "dir/key1"; @@ -169,7 +168,7 @@ public void listMultipartUploadPartsWithoutEtagField() throws IOException { initMultipartUpload(writeClient, volume, bucket, key); - // Commit some MPU parts without eTag field + // Commit some MPU parts, each carrying its (now mandatory) eTag. for (int i = 1; i <= 5; i++) { OmKeyArgs partKeyArgs = new OmKeyArgs.Builder() @@ -199,6 +198,7 @@ public void listMultipartUploadPartsWithoutEtagField() throws IOException { .setReplicationConfig( RatisReplicationConfig.getInstance(ReplicationFactor.THREE)) .setLocationInfoList(Collections.emptyList()) + .addMetadata(OzoneConsts.ETAG, "etag-" + i) .build(); writeClient.commitMultipartUploadPart(commitPartKeyArgs, openKey.getId());
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java index 754fbc1..1ffcc32 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
@@ -61,6 +61,7 @@ import static org.junit.jupiter.params.provider.Arguments.arguments; import java.io.File; +import java.io.IOException; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; @@ -75,6 +76,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.StorageType; @@ -89,8 +91,11 @@ import org.apache.hadoop.ozone.om.helpers.ListOpenFilesResult; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmMultipartPartInfo; +import org.apache.hadoop.ozone.om.helpers.OmMultipartPartKey; import org.apache.hadoop.ozone.om.helpers.OmMultipartUpload; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OpenKeySession; @@ -1047,6 +1052,226 @@ public void testGetExpiredMPUs() throws Exception { assertThat(expiredMPUs).containsAll(names); } + @Test + public void testGetExpiredMPUsSplitSchema() throws Exception { + final String bucketName = UUID.randomUUID().toString(); + final String volumeName = UUID.randomUUID().toString(); + final int numExpiredMPUs = 4; + final int numUnexpiredMPUs = 1; + final int numPartsPerMPU = 5; + final long expireThresholdMillis = ozoneConfiguration.getTimeDuration( + OZONE_OM_MPU_EXPIRE_THRESHOLD, + OZONE_OM_MPU_EXPIRE_THRESHOLD_DEFAULT, + TimeUnit.MILLISECONDS); + + final Duration expireThreshold = Duration.ofMillis(expireThresholdMillis); + + final long expiredMPUCreationTime = + expireThreshold.negated().plusMillis(Time.now()).toMillis(); + + Set<String> expiredMPUs = new HashSet<>(); + for (int i = 0; i < numExpiredMPUs + numUnexpiredMPUs; i++) { + final long creationTime = i < numExpiredMPUs ? + expiredMPUCreationTime : Time.now(); + + String uploadId = OMMultipartUploadUtils.getMultipartUploadId(); + final OmMultipartKeyInfo mpuKeyInfo = new OmMultipartKeyInfo.Builder( + OMRequestTestUtils.createOmMultipartKeyInfo(uploadId, creationTime, + HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.ONE, 0L)) + .setSchemaVersion(OmMultipartKeyInfo.SPLIT_PARTS_TABLE_SCHEMA_VERSION) + .build(); + + String keyName = "expired-split" + i; + final OmKeyInfo keyInfo = OMRequestTestUtils.createOmKeyInfo(volumeName, + bucketName, keyName, RatisReplicationConfig.getInstance(ONE)) + .setCreationTime(creationTime) + .build(); + + for (int j = 1; j <= numPartsPerMPU; j++) { + addSplitSchemaPart(uploadId, j); + } + + final String mpuDbKey = OMRequestTestUtils.addMultipartInfoToTable( + false, keyInfo, mpuKeyInfo, 0L, omMetadataManager); + + expiredMPUs.add(mpuDbKey); + } + + List<ExpiredMultipartUploadsBucket> someExpiredMPUs = + omMetadataManager.getExpiredMultipartUploads( + expireThreshold, + (numExpiredMPUs * numPartsPerMPU) - (numPartsPerMPU)); + List<String> names = getMultipartKeyNames(someExpiredMPUs); + assertEquals(numExpiredMPUs - 1, names.size()); + assertThat(expiredMPUs).containsAll(names); + + List<ExpiredMultipartUploadsBucket> allExpiredMPUs = + omMetadataManager.getExpiredMultipartUploads(expireThreshold, + (numExpiredMPUs * numPartsPerMPU)); + names = getMultipartKeyNames(allExpiredMPUs); + assertEquals(numExpiredMPUs, names.size()); + assertThat(expiredMPUs).containsAll(names); + } + + @Test + public void testGetExpiredMPUsMixedSchema() throws Exception { + final String bucketName = UUID.randomUUID().toString(); + final String volumeName = UUID.randomUUID().toString(); + final int numPartsPerMPU = 3; + final long expireThresholdMillis = ozoneConfiguration.getTimeDuration( + OZONE_OM_MPU_EXPIRE_THRESHOLD, + OZONE_OM_MPU_EXPIRE_THRESHOLD_DEFAULT, + TimeUnit.MILLISECONDS); + final Duration expireThreshold = Duration.ofMillis(expireThresholdMillis); + final long expiredCreationTime = + expireThreshold.negated().plusMillis(Time.now()).toMillis(); + + Set<String> expiredLegacyKeys = new HashSet<>(); + Set<String> expiredSplitKeys = new HashSet<>(); + + // Create 2 legacy-schema expired MPUs with embedded parts + for (int i = 0; i < 2; i++) { + String uploadId = OMMultipartUploadUtils.getMultipartUploadId(); + OmMultipartKeyInfo mpuKeyInfo = OMRequestTestUtils + .createOmMultipartKeyInfo(uploadId, expiredCreationTime, + HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.ONE, 0L); + String keyName = "legacy" + i; + OmKeyInfo keyInfo = OMRequestTestUtils.createOmKeyInfo(volumeName, + bucketName, keyName, RatisReplicationConfig.getInstance(ONE)) + .setCreationTime(expiredCreationTime) + .build(); + for (int j = 1; j <= numPartsPerMPU; j++) { + PartKeyInfo partKeyInfo = OMRequestTestUtils + .createPartKeyInfo(volumeName, bucketName, keyName, uploadId, j); + OMRequestTestUtils.addPart(partKeyInfo, mpuKeyInfo); + } + String mpuDbKey = OMRequestTestUtils.addMultipartInfoToTable( + false, keyInfo, mpuKeyInfo, 0L, omMetadataManager); + expiredLegacyKeys.add(mpuDbKey); + } + + // Create 2 split-schema expired MPUs with parts in multipartPartsTable + for (int i = 0; i < 2; i++) { + String uploadId = OMMultipartUploadUtils.getMultipartUploadId(); + OmMultipartKeyInfo mpuKeyInfo = new OmMultipartKeyInfo.Builder( + OMRequestTestUtils.createOmMultipartKeyInfo(uploadId, expiredCreationTime, + HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.ONE, 0L)) + .setSchemaVersion(OmMultipartKeyInfo.SPLIT_PARTS_TABLE_SCHEMA_VERSION) + .build(); + String keyName = "split" + i; + OmKeyInfo keyInfo = OMRequestTestUtils.createOmKeyInfo(volumeName, + bucketName, keyName, RatisReplicationConfig.getInstance(ONE)) + .setCreationTime(expiredCreationTime) + .build(); + for (int j = 1; j <= numPartsPerMPU; j++) { + addSplitSchemaPart(uploadId, j); + } + String mpuDbKey = OMRequestTestUtils.addMultipartInfoToTable( + false, keyInfo, mpuKeyInfo, 0L, omMetadataManager); + expiredSplitKeys.add(mpuDbKey); + } + + // Budget of 9 parts fits exactly 3 MPUs (3 parts each) + List<ExpiredMultipartUploadsBucket> someExpiredMPUs = + omMetadataManager.getExpiredMultipartUploads(expireThreshold, + numPartsPerMPU * 3); + List<String> names = getMultipartKeyNames(someExpiredMPUs); + assertEquals(3, names.size()); + + // Budget of 12 parts fits all 4 expired MPUs + List<ExpiredMultipartUploadsBucket> allExpiredMPUs = + omMetadataManager.getExpiredMultipartUploads(expireThreshold, + numPartsPerMPU * 4); + names = getMultipartKeyNames(allExpiredMPUs); + assertEquals(4, names.size()); + assertThat(names).containsAll(expiredLegacyKeys); + assertThat(names).containsAll(expiredSplitKeys); + } + + @Test + public void testGetExpiredMPUsZeroPartsSplitSchema() throws Exception { + final String bucketName = UUID.randomUUID().toString(); + final String volumeName = UUID.randomUUID().toString(); + final long expireThresholdMillis = ozoneConfiguration.getTimeDuration( + OZONE_OM_MPU_EXPIRE_THRESHOLD, + OZONE_OM_MPU_EXPIRE_THRESHOLD_DEFAULT, + TimeUnit.MILLISECONDS); + final Duration expireThreshold = Duration.ofMillis(expireThresholdMillis); + final long expiredCreationTime = + expireThreshold.negated().plusMillis(Time.now()).toMillis(); + + // Zero-parts split-schema MPU (freshly initiated, no parts uploaded) + String uploadIdZero = OMMultipartUploadUtils.getMultipartUploadId(); + OmMultipartKeyInfo mpuZero = new OmMultipartKeyInfo.Builder( + OMRequestTestUtils.createOmMultipartKeyInfo(uploadIdZero, expiredCreationTime, + HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.ONE, 0L)) + .setSchemaVersion(OmMultipartKeyInfo.SPLIT_PARTS_TABLE_SCHEMA_VERSION) + .build(); + OmKeyInfo keyInfoZero = OMRequestTestUtils.createOmKeyInfo(volumeName, + bucketName, "aaa-zero-parts", RatisReplicationConfig.getInstance(ONE)) + .setCreationTime(expiredCreationTime) + .build(); + String zeroKey = OMRequestTestUtils.addMultipartInfoToTable( + false, keyInfoZero, mpuZero, 0L, omMetadataManager); + + // Split-schema MPU with 3 parts + String uploadIdThree = OMMultipartUploadUtils.getMultipartUploadId(); + OmMultipartKeyInfo mpuThree = new OmMultipartKeyInfo.Builder( + OMRequestTestUtils.createOmMultipartKeyInfo(uploadIdThree, expiredCreationTime, + HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.ONE, 0L)) + .setSchemaVersion(OmMultipartKeyInfo.SPLIT_PARTS_TABLE_SCHEMA_VERSION) + .build(); + OmKeyInfo keyInfoThree = OMRequestTestUtils.createOmKeyInfo(volumeName, + bucketName, "zzz-three-parts", RatisReplicationConfig.getInstance(ONE)) + .setCreationTime(expiredCreationTime) + .build(); + for (int j = 1; j <= 3; j++) { + addSplitSchemaPart(uploadIdThree, j); + } + String threeKey = OMRequestTestUtils.addMultipartInfoToTable( + false, keyInfoThree, mpuThree, 0L, omMetadataManager); + + // maxParts=0 returns nothing (loop pre-check 0 < 0 is false) + List<ExpiredMultipartUploadsBucket> noneExpired = + omMetadataManager.getExpiredMultipartUploads(expireThreshold, 0); + assertTrue(getMultipartKeyNames(noneExpired).isEmpty()); + + // maxParts=3 should return both: zero-parts contributes 0, three-parts + // contributes 3, total = 3 which satisfies the loop exit condition + List<ExpiredMultipartUploadsBucket> allExpired = + omMetadataManager.getExpiredMultipartUploads(expireThreshold, 3); + List<String> names = getMultipartKeyNames(allExpired); + assertEquals(2, names.size()); + assertThat(names).contains(zeroKey); + assertThat(names).contains(threeKey); + } + + private void addSplitSchemaPart(String uploadId, int partNumber) throws IOException { + OmKeyLocationInfo locationInfo = new OmKeyLocationInfo.Builder() + .setBlockID(new BlockID(1L, partNumber)) + .setLength(100) + .build(); + OmKeyLocationInfoGroup locationGroup = new OmKeyLocationInfoGroup(0, + Collections.singletonList(locationInfo)); + String partName = "part-" + partNumber; + OmKeyInfo keyInfo = OMRequestTestUtils.createOmKeyInfo("vol", "bucket", "key", + RatisReplicationConfig.getInstance(ONE)) + .setDataSize(100L) + .setObjectID(partNumber) + .setUpdateID(partNumber) + .addOmKeyLocationInfoGroup(locationGroup) + .addMetadata(org.apache.hadoop.ozone.OzoneConsts.ETAG, "etag-" + partNumber) + .build(); + OmMultipartPartInfo partInfo = OmMultipartPartInfo.from(partName, partNumber, keyInfo); + omMetadataManager.getMultipartPartsTable().put( + OmMultipartPartKey.of(uploadId, partNumber), partInfo); + } + private List<String> getOpenKeyNames( Collection<OpenKeyBucket.Builder> openKeyBuckets) { return openKeyBuckets.stream()
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/lock/TestKeyPathLock.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/lock/TestKeyPathLock.java index 3122f65..77b7999 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/lock/TestKeyPathLock.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/lock/TestKeyPathLock.java
@@ -216,7 +216,7 @@ private void testDiffKeyPathWriteLockMultiThreadingUtil( // Waiting for all the threads to be instantiated/to reach // acquireWriteLock. countDown.countDown(); - assertEquals(1, lock.getCurrentLocks().size()); + assertEquals(1, lock.getCurrentLockSizeForTesting()); lock.releaseWriteLock(resource, sampleResourceName); LOG.info("Write Lock Released by " + Thread.currentThread().getName());
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequest.java index 8f720de..58949fe 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequest.java
@@ -20,13 +20,41 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import jakarta.annotation.Nonnull; +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.UUID; +import org.apache.hadoop.hdds.client.ContainerBlockID; import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.MockDatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; +import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock; +import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; +import org.apache.hadoop.hdds.scm.net.NetworkTopology; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; +import org.apache.hadoop.hdds.scm.pipeline.PipelineID; +import org.apache.hadoop.ipc_.Server; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.KeyManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; @@ -36,7 +64,11 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.AllocateBlockRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.UserInfo; +import org.apache.hadoop.security.UserGroupInformation; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.MockedStatic; /** * Tests OMAllocateBlockRequest class. @@ -225,6 +257,253 @@ protected OMRequest doPreExecute(OMRequest originalOMRequest) return modifiedOmRequest; } + @Test + public void testAllocateBlockSendsClientMachineToScmWhenFlagOff() throws Exception { + // Flag off (default): OM must NOT sort; SCM receives the real client address + // so it performs the sort. + KeyManager mockKeyManager = mock(KeyManager.class); + when(mockKeyManager.isSortDatanodesForWriteEnabled()).thenReturn(false); + when(ozoneManager.getKeyManager()).thenReturn(mockKeyManager); + + OMAllocateBlockRequest request = + getOmAllocateBlockRequest(createAllocateBlockRequestWithSort()); + preExecuteWithClient(request, "1.2.3.4"); + + ArgumentCaptor<String> clientMachine = ArgumentCaptor.forClass(String.class); + verify(scmBlockLocationProtocol).allocateBlock(anyLong(), anyInt(), any(), + any(), any(), clientMachine.capture()); + assertEquals("1.2.3.4", clientMachine.getValue()); + verify(mockKeyManager, never()).sortDatanodesForWrite(any(), anyString(), any()); + } + + @Test + public void testAllocateBlockDoesNotSendClientMachineToScm() throws Exception { + // OM now sorts the write pipeline locally, so SCM must receive an empty + // clientMachine even when the client requests sorted datanodes. + KeyManager mockKeyManager = mock(KeyManager.class); + when(mockKeyManager.isSortDatanodesForWriteEnabled()).thenReturn(true); + when(mockKeyManager.sortDatanodesForWrite(any(), any(), any())) + .thenAnswer(inv -> inv.getArgument(0)); + when(ozoneManager.getKeyManager()).thenReturn(mockKeyManager); + when(ozoneManager.getClusterMapAllowNull()).thenReturn(mock(NetworkTopology.class)); + + OMAllocateBlockRequest request = + getOmAllocateBlockRequest(createAllocateBlockRequestWithSort()); + preExecuteWithClient(request, "1.2.3.4"); + + ArgumentCaptor<String> clientMachine = ArgumentCaptor.forClass(String.class); + verify(scmBlockLocationProtocol).allocateBlock(anyLong(), anyInt(), any(), + any(), any(), clientMachine.capture()); + assertEquals("", clientMachine.getValue()); + } + + @Test + public void testAllocateBlockFallsBackToScmWhenTopologyUnavailable() throws Exception { + KeyManager mockKeyManager = mock(KeyManager.class); + when(mockKeyManager.isSortDatanodesForWriteEnabled()).thenReturn(true); + when(ozoneManager.getKeyManager()).thenReturn(mockKeyManager); + + OMAllocateBlockRequest request = + getOmAllocateBlockRequest(createAllocateBlockRequestWithSort()); + preExecuteWithClient(request, "1.2.3.4"); + + ArgumentCaptor<String> clientMachine = ArgumentCaptor.forClass(String.class); + verify(scmBlockLocationProtocol).allocateBlock(anyLong(), anyInt(), any(), + any(), any(), clientMachine.capture()); + assertEquals("1.2.3.4", clientMachine.getValue()); + verify(mockKeyManager, never()).sortDatanodesForWrite(any(), anyString(), any()); + } + + @Test + public void testAllocateBlockSortsSharedPipelineOnce() throws Exception { + // Two blocks on the same 3-node pipeline must be sorted once, and the + // sorted order must land in every block's pipeline. + List<DatanodeDetails> nodes = Arrays.asList( + MockDatanodeDetails.randomDatanodeDetails(), + MockDatanodeDetails.randomDatanodeDetails(), + MockDatanodeDetails.randomDatanodeDetails()); + Pipeline pipeline = Pipeline.newBuilder() + .setState(Pipeline.PipelineState.OPEN) + .setId(PipelineID.randomId()) + .setReplicationConfig( + StandaloneReplicationConfig.getInstance(ReplicationFactor.THREE)) + .setNodes(nodes) + .build(); + AllocatedBlock.Builder blockBuilder = + new AllocatedBlock.Builder().setPipeline(pipeline); + when(scmBlockLocationProtocol.allocateBlock(anyLong(), anyInt(), any(), + anyString(), any(ExcludeList.class), anyString())).thenAnswer(inv -> { + int num = inv.getArgument(1); + List<AllocatedBlock> blocks = new ArrayList<>(num); + for (int i = 0; i < num; i++) { + blockBuilder.setContainerBlockID( + new ContainerBlockID(CONTAINER_ID + i, LOCAL_ID + i)); + blocks.add(blockBuilder.build()); + } + return blocks; + }); + + List<DatanodeDetails> sortedOrder = new ArrayList<>(nodes); + Collections.reverse(sortedOrder); + KeyManager mockKeyManager = mock(KeyManager.class); + when(mockKeyManager.isSortDatanodesForWriteEnabled()).thenReturn(true); + when(mockKeyManager.sortDatanodesForWrite(any(), any(), any())) + .thenAnswer(inv -> sortedOrder); + when(ozoneManager.getKeyManager()).thenReturn(mockKeyManager); + when(ozoneManager.getClusterMapAllowNull()).thenReturn(mock(NetworkTopology.class)); + + OMAllocateBlockRequest request = + getOmAllocateBlockRequest(createAllocateBlockRequest()); + // requestedSize spans two scmBlockSize blocks on the same pipeline. + List<OmKeyLocationInfo> locations = request.allocateBlock(replicationConfig, + new ExcludeList(), 2 * scmBlockSize, true, + UserInfo.newBuilder().setRemoteAddress("1.2.3.4").build(), ozoneManager); + + // Sorted once for the shared pipeline... + verify(mockKeyManager, times(1)).sortDatanodesForWrite(any(), eq("1.2.3.4"), any()); + // ...and the sorted order is applied to every block's pipeline. + assertEquals(2, locations.size()); + for (OmKeyLocationInfo location : locations) { + assertEquals(sortedOrder, location.getPipeline().getNodesInOrder()); + } + } + + @Test + public void testAllocateBlockKeepsPerPipelineOrderWhenSortSkipped() throws Exception { + // Two pipelines share the same datanode set but in a different order. When + // the sort is skipped (sortDatanodesForWrite returns the input unchanged), + // each pipeline must keep its own order: the unsorted result must not be + // cached under the node set and reused for the other pipeline. + DatanodeDetails a = MockDatanodeDetails.randomDatanodeDetails(); + DatanodeDetails b = MockDatanodeDetails.randomDatanodeDetails(); + DatanodeDetails c = MockDatanodeDetails.randomDatanodeDetails(); + List<DatanodeDetails> nodes1 = Arrays.asList(a, b, c); + List<DatanodeDetails> nodes2 = Arrays.asList(c, b, a); + Pipeline pipeline1 = Pipeline.newBuilder() + .setState(Pipeline.PipelineState.OPEN) + .setId(PipelineID.randomId()) + .setReplicationConfig( + StandaloneReplicationConfig.getInstance(ReplicationFactor.THREE)) + .setNodes(nodes1) + .build(); + Pipeline pipeline2 = Pipeline.newBuilder() + .setState(Pipeline.PipelineState.OPEN) + .setId(PipelineID.randomId()) + .setReplicationConfig( + StandaloneReplicationConfig.getInstance(ReplicationFactor.THREE)) + .setNodes(nodes2) + .build(); + AllocatedBlock block1 = new AllocatedBlock.Builder().setPipeline(pipeline1) + .setContainerBlockID(new ContainerBlockID(CONTAINER_ID, LOCAL_ID)).build(); + AllocatedBlock block2 = new AllocatedBlock.Builder().setPipeline(pipeline2) + .setContainerBlockID(new ContainerBlockID(CONTAINER_ID + 1, LOCAL_ID + 1)).build(); + when(scmBlockLocationProtocol.allocateBlock(anyLong(), anyInt(), any(), + anyString(), any(ExcludeList.class), anyString())) + .thenReturn(Arrays.asList(block1, block2)); + + KeyManager mockKeyManager = mock(KeyManager.class); + when(mockKeyManager.isSortDatanodesForWriteEnabled()).thenReturn(true); + // Skip the sort: return the input list instance unchanged. + when(mockKeyManager.sortDatanodesForWrite(any(), any(), any())) + .thenAnswer(inv -> inv.getArgument(0)); + when(ozoneManager.getKeyManager()).thenReturn(mockKeyManager); + when(ozoneManager.getClusterMapAllowNull()).thenReturn(mock(NetworkTopology.class)); + + OMAllocateBlockRequest request = + getOmAllocateBlockRequest(createAllocateBlockRequest()); + List<OmKeyLocationInfo> locations = request.allocateBlock(replicationConfig, + new ExcludeList(), 2 * scmBlockSize, true, + UserInfo.newBuilder().setRemoteAddress("1.2.3.4").build(), ozoneManager); + + assertEquals(2, locations.size()); + // Each pipeline keeps its own order; the skipped-sort result is not shared. + assertEquals(nodes1, locations.get(0).getPipeline().getNodesInOrder()); + assertEquals(nodes2, locations.get(1).getPipeline().getNodesInOrder()); + // Sorted per pipeline, since the unsorted result is not cached. + verify(mockKeyManager, times(2)).sortDatanodesForWrite(any(), eq("1.2.3.4"), any()); + } + + @Test + public void testAllocateBlockKeepsOrderWhenRemoteAddressEmpty() throws Exception { + // Sort enabled and topology available, but the client has no remote address: + // OM must not sort, SCM receives an empty clientMachine, and the pipeline + // order is preserved. + List<DatanodeDetails> nodes = Arrays.asList( + MockDatanodeDetails.randomDatanodeDetails(), + MockDatanodeDetails.randomDatanodeDetails(), + MockDatanodeDetails.randomDatanodeDetails()); + Pipeline pipeline = Pipeline.newBuilder() + .setState(Pipeline.PipelineState.OPEN) + .setId(PipelineID.randomId()) + .setReplicationConfig( + StandaloneReplicationConfig.getInstance(ReplicationFactor.THREE)) + .setNodes(nodes) + .build(); + AllocatedBlock block = new AllocatedBlock.Builder().setPipeline(pipeline) + .setContainerBlockID(new ContainerBlockID(CONTAINER_ID, LOCAL_ID)).build(); + ArgumentCaptor<String> clientMachine = ArgumentCaptor.forClass(String.class); + when(scmBlockLocationProtocol.allocateBlock(anyLong(), anyInt(), any(), + anyString(), any(ExcludeList.class), clientMachine.capture())) + .thenReturn(Collections.singletonList(block)); + + KeyManager mockKeyManager = mock(KeyManager.class); + when(mockKeyManager.isSortDatanodesForWriteEnabled()).thenReturn(true); + when(ozoneManager.getKeyManager()).thenReturn(mockKeyManager); + when(ozoneManager.getClusterMapAllowNull()).thenReturn(mock(NetworkTopology.class)); + + OMAllocateBlockRequest request = + getOmAllocateBlockRequest(createAllocateBlockRequest()); + List<OmKeyLocationInfo> locations = request.allocateBlock(replicationConfig, + new ExcludeList(), scmBlockSize, true, + UserInfo.newBuilder().setRemoteAddress("").build(), ozoneManager); + + assertEquals("", clientMachine.getValue()); + verify(mockKeyManager, never()).sortDatanodesForWrite(any(), anyString(), any()); + assertEquals(1, locations.size()); + // Assert the write order (nodesInOrder), which copyWithNodesInOrder would + // have changed had OM sorted; it must stay as the original pipeline order. + assertEquals(nodes, locations.get(0).getPipeline().getNodesInOrder()); + } + + @Test + public void sortDatanodesForWriteRequiresClientMachine() { + List<DatanodeDetails> nodes = Arrays.asList( + MockDatanodeDetails.randomDatanodeDetails(), + MockDatanodeDetails.randomDatanodeDetails(), + MockDatanodeDetails.randomDatanodeDetails()); + assertThrows(IllegalArgumentException.class, + () -> keyManager.sortDatanodesForWrite(nodes, "", mock(NetworkTopology.class))); + } + + // Like createAllocateBlockRequest, but sets sortDatanodes so preExecute + // resolves the client address from the RPC context. + private OMRequest createAllocateBlockRequestWithSort() { + KeyArgs keyArgs = KeyArgs.newBuilder() + .setVolumeName(volumeName).setBucketName(bucketName).setKeyName(keyName) + .setFactor(((RatisReplicationConfig) replicationConfig).getReplicationFactor()) + .setType(replicationConfig.getReplicationType()) + .setSortDatanodes(true) + .build(); + AllocateBlockRequest allocateBlockRequest = AllocateBlockRequest.newBuilder() + .setClientID(clientID).setKeyArgs(keyArgs).build(); + return OMRequest.newBuilder() + .setCmdType(OzoneManagerProtocolProtos.Type.AllocateBlock) + .setClientId(UUID.randomUUID().toString()) + .setAllocateBlockRequest(allocateBlockRequest).build(); + } + + // Run preExecute with a mocked RPC context so UserInfo carries clientAddress, + // the way an OM RPC handler thread would see it. + private void preExecuteWithClient(OMAllocateBlockRequest request, String clientAddress) throws Exception { + InetAddress clientIp = InetAddress.getByAddress(clientAddress, InetAddress.getByName(clientAddress).getAddress()); + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + try (MockedStatic<Server> mockedRpcServer = mockStatic(Server.class)) { + mockedRpcServer.when(Server::getRemoteUser).thenReturn(ugi); + mockedRpcServer.when(Server::getRemoteIp).thenReturn(clientIp); + request.preExecute(ozoneManager); + } + } + protected OMRequest createAllocateBlockRequest() { KeyArgs keyArgs = KeyArgs.newBuilder()
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartRequestTests.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartRequestTests.java index e23b84f..08e8487 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartRequestTests.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartRequestTests.java
@@ -34,6 +34,8 @@ import java.util.Map; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.db.cache.CacheKey; +import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.audit.AuditLogger; import org.apache.hadoop.ozone.audit.AuditMessage; import org.apache.hadoop.ozone.om.IOmMetadataReader; @@ -47,8 +49,11 @@ import org.apache.hadoop.ozone.om.ResolvedBucket; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.KeyValueUtil; +import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyLocation; @@ -108,6 +113,11 @@ public void setup() throws Exception { args.getVolumeName(), args.getBucketName(), "owner", BucketLayout.DEFAULT); }); + // MPU request tests default to a pre-finalized layout version, i.e. a + // cluster that has not finalized MPU_PARTS_TABLE_SPLIT. Newly initiated + // uploads therefore use the legacy (schema 0) inline parts layout. Tests + // that need the finalized behaviour (split parts table, schema 1) opt in + // by calling finalizeMpuPartsTableSplit(). OMLayoutVersionManager lvm = mock(OMLayoutVersionManager.class); when(lvm.getMetadataLayoutVersion()).thenReturn(0); when(ozoneManager.getVersionManager()).thenReturn(lvm); @@ -115,6 +125,18 @@ public void setup() throws Exception { when(ozoneManager.getConfig()).thenReturn(ozoneConfiguration.getObject(OmConfig.class)); } + /** + * Simulate a cluster that has finalized the multipart parts-table split + * layout feature, so newly initiated uploads resolve to the split (schema 1) + * parts-table layout. This stubs the exact signal the request path checks + * ({@code isAllowed(MPU_PARTS_TABLE_SPLIT)}) rather than the metadata layout + * version, which the MPU schema gate does not read directly. + */ + protected void finalizeMpuPartsTableSplit() { + when(ozoneManager.getVersionManager() + .isAllowed(OMLayoutFeature.MPU_PARTS_TABLE_SPLIT)).thenReturn(true); + } + @AfterEach public void stop() { omMetrics.unRegister(); @@ -303,6 +325,46 @@ protected OMRequest doPreExecuteCompleteMPU( } /** + * Initiate an MPU and optionally rewrite the stored multipart metadata to a + * specific schema version. + * + * <p>The schema version rewrite lets tests emulate post-finalization MPU + * entries without needing the rest of the upgrade pipeline.</p> + */ + protected String initiateMultipartUploadWithSchemaVersion( + String volumeName, String bucketName, String keyName, + int schemaVersion) throws Exception { + OMRequest initiateMPURequest = + doPreExecuteInitiateMPU(volumeName, bucketName, keyName); + + S3InitiateMultipartUploadRequest s3InitiateMultipartUploadRequest = + getS3InitiateMultipartUploadReq(initiateMPURequest); + + OMClientResponse omClientResponse = + s3InitiateMultipartUploadRequest.validateAndUpdateCache(ozoneManager, + 1L); + + String multipartUploadID = omClientResponse.getOMResponse() + .getInitiateMultiPartUploadResponse().getMultipartUploadID(); + + if (schemaVersion != 0) { + String multipartKey = omMetadataManager.getMultipartKey(volumeName, + bucketName, keyName, multipartUploadID); + OmMultipartKeyInfo multipartKeyInfo = omMetadataManager + .getMultipartInfoTable().get(multipartKey); + assertNotNull(multipartKeyInfo); + + omMetadataManager.getMultipartInfoTable().addCacheEntry( + new CacheKey<>(multipartKey), + CacheValue.get(2L, multipartKeyInfo.toBuilder() + .setSchemaVersion(schemaVersion) + .build())); + } + + return multipartUploadID; + } + + /** * Perform preExecute of Initiate Multipart upload request for given * volume, bucket and key name. * @param volumeName
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequest.java index 79cf874..e316fd4 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequest.java
@@ -24,6 +24,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -32,11 +33,14 @@ import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; /** * Tests S3 Initiate Multipart Upload request. @@ -113,6 +117,110 @@ public void testValidateAndUpdateCache() throws Exception { } + /** + * The schema version is a server-owned decision resolved in {@code preExecute} + * (on the leader) from the finalized layout version; {@code + * validateAndUpdateCache} only forwards the stamped value into the persisted + * multipart info row. Pre-finalization -> legacy (0); finalized -> split (1). + */ + @ParameterizedTest + @CsvSource({ + // finalized, expectedSchemaVersion (0 = LEGACY, 1 = SPLIT_PARTS_TABLE) + "false, 0", + "true, 1", + }) + public void testSchemaVersionStampedInPreExecuteByServer( + boolean finalized, int expectedSchemaVersion) throws Exception { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + String keyName = UUID.randomUUID().toString(); + + if (finalized) { + finalizeMpuPartsTableSplit(); + } + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, getBucketLayout()); + + // preExecute (leader) stamps the schema version onto the request. + OMRequest modifiedRequest = doPreExecuteInitiateMPU(volumeName, + bucketName, keyName); + assertEquals(expectedSchemaVersion, + modifiedRequest.getInitiateMultiPartUploadRequest().getSchemaVersion()); + + // validateAndUpdateCache only forwards the already-stamped value. + OMClientResponse response = getS3InitiateMultipartUploadReq(modifiedRequest) + .validateAndUpdateCache(ozoneManager, 100L); + assertEquals(OzoneManagerProtocolProtos.Status.OK, + response.getOMResponse().getStatus()); + + String multipartKey = getMultipartKey(volumeName, bucketName, keyName, + modifiedRequest.getInitiateMultiPartUploadRequest() + .getKeyArgs().getMultipartUploadID()); + OmMultipartKeyInfo multipartKeyInfo = omMetadataManager + .getMultipartInfoTable().get(multipartKey); + assertNotNull(multipartKeyInfo); + assertEquals(expectedSchemaVersion, multipartKeyInfo.getSchemaVersion()); + } + + /** + * The schema version is server-owned: a client-supplied value on the request + * is ignored and overwritten in {@code preExecute} with the server decision, + * in both directions (client asks for split pre-finalization, and client asks + * for legacy post-finalization). + */ + @ParameterizedTest + @CsvSource({ + // finalized, expectedSchemaVersion (0 = LEGACY, 1 = SPLIT_PARTS_TABLE) + "false, 0", + "true, 1", + }) + public void testServerIgnoresClientSuppliedSchemaVersion( + boolean finalized, int expectedSchemaVersion) throws Exception { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + String keyName = UUID.randomUUID().toString(); + + if (finalized) { + finalizeMpuPartsTableSplit(); + } + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, getBucketLayout()); + + // Client supplies the opposite of what the server should decide. + int clientSuppliedSchemaVersion = finalized + ? OmMultipartKeyInfo.LEGACY_SCHEMA_VERSION + : OmMultipartKeyInfo.SPLIT_PARTS_TABLE_SCHEMA_VERSION; + OMRequest clientRequest = OMRequestTestUtils.createInitiateMPURequest( + volumeName, bucketName, keyName, Collections.emptyMap(), + Collections.emptyMap()); + clientRequest = clientRequest.toBuilder() + .setInitiateMultiPartUploadRequest( + clientRequest.getInitiateMultiPartUploadRequest().toBuilder() + .setSchemaVersion(clientSuppliedSchemaVersion)) + .build(); + + // preExecute must overwrite the client value with the server decision. + OMRequest modifiedRequest = + getS3InitiateMultipartUploadReq(clientRequest).preExecute(ozoneManager); + assertEquals(expectedSchemaVersion, + modifiedRequest.getInitiateMultiPartUploadRequest().getSchemaVersion()); + + OMClientResponse response = getS3InitiateMultipartUploadReq(modifiedRequest) + .validateAndUpdateCache(ozoneManager, 100L); + assertEquals(OzoneManagerProtocolProtos.Status.OK, + response.getOMResponse().getStatus()); + + String multipartKey = getMultipartKey(volumeName, bucketName, keyName, + modifiedRequest.getInitiateMultiPartUploadRequest() + .getKeyArgs().getMultipartUploadID()); + OmMultipartKeyInfo multipartKeyInfo = omMetadataManager + .getMultipartInfoTable().get(multipartKey); + assertNotNull(multipartKeyInfo); + assertEquals(expectedSchemaVersion, multipartKeyInfo.getSchemaVersion()); + } + @Test public void testValidateAndUpdateCacheWithBucketNotFound() throws Exception { String volumeName = UUID.randomUUID().toString();
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadAbortRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadAbortRequest.java index 53ae47f..3d529ad 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadAbortRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadAbortRequest.java
@@ -25,6 +25,7 @@ import java.util.UUID; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -96,6 +97,93 @@ public void testValidateAndUpdateCache() throws Exception { } @Test + public void testValidateAndUpdateCacheUsesSchemaVersionOneBeforeFinalization() + throws Exception { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + String keyName = getKeyName(); + + // The base test fixture is pre-finalized by default. + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, getBucketLayout()); + + createParentPath(volumeName, bucketName); + + String multipartUploadID = + initiateMultipartUploadWithSchemaVersion(volumeName, bucketName, + keyName, 1); + + String multipartKey = omMetadataManager.getMultipartKey(volumeName, + bucketName, keyName, multipartUploadID); + OmMultipartKeyInfo multipartKeyInfo = omMetadataManager + .getMultipartInfoTable().get(multipartKey); + assertNotNull(multipartKeyInfo); + assertEquals(1, multipartKeyInfo.getSchemaVersion()); + + OMRequest abortMPURequest = + doPreExecuteAbortMPU(volumeName, bucketName, keyName, + multipartUploadID); + + S3MultipartUploadAbortRequest s3MultipartUploadAbortRequest = + getS3MultipartUploadAbortReq(abortMPURequest); + + OMClientResponse omClientResponse = + s3MultipartUploadAbortRequest.validateAndUpdateCache(ozoneManager, 2L); + + assertEquals(OzoneManagerProtocolProtos.Status.OK, + omClientResponse.getOMResponse().getStatus()); + } + + @Test + public void testValidateAndUpdateCacheAllowsSchemaVersionZeroAfterFinalization() + throws Exception { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + String keyName = getKeyName(); + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, getBucketLayout()); + + createParentPath(volumeName, bucketName); + + // Upload is initiated on a pre-finalized cluster, so it uses the legacy + // (schema 0) inline layout. + OMRequest initiateMPURequest = doPreExecuteInitiateMPU(volumeName, + bucketName, keyName); + S3InitiateMultipartUploadRequest s3InitiateMultipartUploadRequest = + getS3InitiateMultipartUploadReq(initiateMPURequest); + OMClientResponse initiateResponse = + s3InitiateMultipartUploadRequest.validateAndUpdateCache(ozoneManager, + 1L); + String multipartUploadID = initiateResponse.getOMResponse() + .getInitiateMultiPartUploadResponse().getMultipartUploadID(); + + String multipartKey = omMetadataManager.getMultipartKey(volumeName, + bucketName, keyName, multipartUploadID); + OmMultipartKeyInfo multipartKeyInfo = omMetadataManager + .getMultipartInfoTable().get(multipartKey); + assertNotNull(multipartKeyInfo); + assertEquals(0, multipartKeyInfo.getSchemaVersion()); + + // Cluster finalizes the split feature, the pre-existing legacy upload must + // still be abortable. + finalizeMpuPartsTableSplit(); + + OMRequest abortMPURequest = + doPreExecuteAbortMPU(volumeName, bucketName, keyName, + multipartUploadID); + + S3MultipartUploadAbortRequest s3MultipartUploadAbortRequest = + getS3MultipartUploadAbortReq(abortMPURequest); + + OMClientResponse omClientResponse = + s3MultipartUploadAbortRequest.validateAndUpdateCache(ozoneManager, 2L); + + assertEquals(OzoneManagerProtocolProtos.Status.OK, + omClientResponse.getOMResponse().getStatus()); + } + + @Test public void testValidateAndUpdateCacheMultipartNotFound() throws Exception { String volumeName = UUID.randomUUID().toString(); String bucketName = UUID.randomUUID().toString();
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadCommitPartRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadCommitPartRequest.java index b451300..e9ccbb4 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadCommitPartRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadCommitPartRequest.java
@@ -39,7 +39,6 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; @@ -140,6 +139,100 @@ public void testValidateAndUpdateCacheSuccess() throws Exception { } @Test + public void testValidateAndUpdateCacheUsesSchemaVersionOneBeforeFinalization() + throws Exception { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + String keyName = getKeyName(); + + // The base test fixture is pre-finalized by default. + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, getBucketLayout()); + + createParentPath(volumeName, bucketName); + + String multipartUploadID = + initiateMultipartUploadWithSchemaVersion(volumeName, bucketName, + keyName, OmMultipartKeyInfo.SPLIT_PARTS_TABLE_SCHEMA_VERSION); + + String multipartKey = omMetadataManager.getMultipartKey(volumeName, + bucketName, keyName, multipartUploadID); + OmMultipartKeyInfo multipartKeyInfo = omMetadataManager + .getMultipartInfoTable().get(multipartKey); + assertNotNull(multipartKeyInfo); + assertEquals(OmMultipartKeyInfo.SPLIT_PARTS_TABLE_SCHEMA_VERSION, + multipartKeyInfo.getSchemaVersion()); + + long clientID = Time.now(); + OMRequest commitMultipartRequest = doPreExecuteCommitMPU(volumeName, + bucketName, keyName, clientID, multipartUploadID, 1); + + S3MultipartUploadCommitPartRequest s3MultipartUploadCommitPartRequest = + getS3MultipartUploadCommitReq(commitMultipartRequest); + + addKeyToOpenKeyTable(volumeName, bucketName, keyName, clientID); + + OMClientResponse omClientResponse = + s3MultipartUploadCommitPartRequest.validateAndUpdateCache(ozoneManager, + 2L); + + assertEquals(OzoneManagerProtocolProtos.Status.OK, + omClientResponse.getOMResponse().getStatus()); + } + + @Test + public void testValidateAndUpdateCacheAllowsSchemaVersionZeroAfterFinalization() + throws Exception { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + String keyName = getKeyName(); + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, getBucketLayout()); + + createParentPath(volumeName, bucketName); + + // Upload is initiated on a pre-finalized cluster, so it uses the legacy + // (schema 0) inline layout. + OMRequest initiateMPURequest = doPreExecuteInitiateMPU(volumeName, + bucketName, keyName); + S3InitiateMultipartUploadRequest s3InitiateMultipartUploadRequest = + getS3InitiateMultipartUploadReq(initiateMPURequest); + OMClientResponse initiateResponse = + s3InitiateMultipartUploadRequest.validateAndUpdateCache(ozoneManager, + 1L); + String multipartUploadID = initiateResponse.getOMResponse() + .getInitiateMultiPartUploadResponse().getMultipartUploadID(); + + String multipartKey = omMetadataManager.getMultipartKey(volumeName, + bucketName, keyName, multipartUploadID); + OmMultipartKeyInfo multipartKeyInfo = omMetadataManager + .getMultipartInfoTable().get(multipartKey); + assertNotNull(multipartKeyInfo); + assertEquals(0, multipartKeyInfo.getSchemaVersion()); + + // Cluster finalizes the split feature; the pre-existing legacy part must + // still be committable. + finalizeMpuPartsTableSplit(); + + long clientID = Time.now(); + OMRequest commitMultipartRequest = doPreExecuteCommitMPU(volumeName, + bucketName, keyName, clientID, multipartUploadID, 1); + + S3MultipartUploadCommitPartRequest s3MultipartUploadCommitPartRequest = + getS3MultipartUploadCommitReq(commitMultipartRequest); + + addKeyToOpenKeyTable(volumeName, bucketName, keyName, clientID); + + OMClientResponse omClientResponse = + s3MultipartUploadCommitPartRequest.validateAndUpdateCache(ozoneManager, + 2L); + + assertEquals(OzoneManagerProtocolProtos.Status.OK, + omClientResponse.getOMResponse().getStatus()); + } + + @Test public void testValidateAndUpdateCacheMultipartNotFound() throws Exception { String volumeName = UUID.randomUUID().toString(); String bucketName = UUID.randomUUID().toString(); @@ -172,7 +265,6 @@ public void testValidateAndUpdateCacheMultipartNotFound() throws Exception { bucketName, keyName, multipartUploadID); assertNull(omMetadataManager.getMultipartInfoTable().get(multipartKey)); - } @Test @@ -184,9 +276,19 @@ public void testValidateAndUpdateCacheKeyNotFound() throws Exception { OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, getBucketLayout()); + createParentPath(volumeName, bucketName); + + OMRequest initiateMPURequest = doPreExecuteInitiateMPU(volumeName, + bucketName, keyName); + S3InitiateMultipartUploadRequest s3InitiateMultipartUploadRequest = + getS3InitiateMultipartUploadReq(initiateMPURequest); + OMClientResponse initiateResponse = + s3InitiateMultipartUploadRequest.validateAndUpdateCache(ozoneManager, + 1L); long clientID = Time.now(); - String multipartUploadID = UUID.randomUUID().toString(); + String multipartUploadID = initiateResponse.getOMResponse() + .getInitiateMultiPartUploadResponse().getMultipartUploadID(); OMRequest commitMultipartRequest = doPreExecuteCommitMPU(volumeName, bucketName, keyName, clientID, multipartUploadID, 1); @@ -201,13 +303,8 @@ public void testValidateAndUpdateCacheKeyNotFound() throws Exception { OMClientResponse omClientResponse = s3MultipartUploadCommitPartRequest.validateAndUpdateCache(ozoneManager, 2L); - if (getBucketLayout() == BucketLayout.FILE_SYSTEM_OPTIMIZED) { - assertSame(omClientResponse.getOMResponse().getStatus(), - OzoneManagerProtocolProtos.Status.DIRECTORY_NOT_FOUND); - } else { - assertSame(omClientResponse.getOMResponse().getStatus(), - OzoneManagerProtocolProtos.Status.KEY_NOT_FOUND); - } + assertSame(omClientResponse.getOMResponse().getStatus(), + OzoneManagerProtocolProtos.Status.KEY_NOT_FOUND); } @@ -742,7 +839,13 @@ public void testSplitSchemaCommitFailsWithoutETag() throws Exception { S3MultipartUploadCommitPartRequest request = getS3MultipartUploadCommitReq(omRequest); OMClientResponse response = request.validateAndUpdateCache(ozoneManager, 2L); - assertSame(OzoneManagerProtocolProtos.Status.INVALID_REQUEST, response.getOMResponse().getStatus()); + assertSame(OzoneManagerProtocolProtos.Status.INVALID_REQUEST, + response.getOMResponse().getStatus()); + + // No part row should have been written to the split parts table. + SortedMap<Integer, OmMultipartPartInfo> parts = + OMMultipartUploadUtils.scanParts(omMetadataManager, uploadId); + assertEquals(0, parts.size()); } @Test
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadCompleteRequest.java index 653b070..27b7217 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadCompleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadCompleteRequest.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.om.response.OMClientResponse; @@ -206,6 +207,82 @@ private String checkValidateAndUpdateCacheSuccess(String volumeName, return multipartUploadID; } + @Test + public void testValidateAndUpdateCacheUsesSchemaVersionOneBeforeFinalization() + throws Exception { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + String keyName = getKeyName(); + + // The base test fixture is pre-finalized by default. + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, getBucketLayout()); + + String multipartUploadID = + initiateMultipartUploadWithSchemaVersion(volumeName, bucketName, + keyName, 1); + + OMRequest completeMultipartRequest = doPreExecuteCompleteMPU(volumeName, + bucketName, keyName, multipartUploadID, new ArrayList<>()); + + S3MultipartUploadCompleteRequest s3MultipartUploadCompleteRequest = + getS3MultipartUploadCompleteReq(completeMultipartRequest); + + OMClientResponse omClientResponse = + s3MultipartUploadCompleteRequest.validateAndUpdateCache(ozoneManager, + 3L); + + assertEquals(OzoneManagerProtocolProtos.Status.INVALID_REQUEST, + omClientResponse.getOMResponse().getStatus()); + } + + @Test + public void testValidateAndUpdateCacheAllowsSchemaVersionZeroAfterFinalization() + throws Exception { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + String keyName = getKeyName(); + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, getBucketLayout()); + + // Upload is initiated on a pre-finalized cluster, so it uses the legacy + // (schema 0) inline layout. + OMRequest initiateMPURequest = doPreExecuteInitiateMPU(volumeName, + bucketName, keyName); + S3InitiateMultipartUploadRequest s3InitiateMultipartUploadRequest = + getS3InitiateMultipartUploadReq(initiateMPURequest); + OMClientResponse initiateResponse = + s3InitiateMultipartUploadRequest.validateAndUpdateCache(ozoneManager, + 1L); + String multipartUploadID = initiateResponse.getOMResponse() + .getInitiateMultiPartUploadResponse().getMultipartUploadID(); + + String multipartKey = getMultipartKey(volumeName, bucketName, keyName, + multipartUploadID); + OmMultipartKeyInfo multipartKeyInfo = omMetadataManager + .getMultipartInfoTable().get(multipartKey); + assertNotNull(multipartKeyInfo); + assertEquals(0, multipartKeyInfo.getSchemaVersion()); + + // Cluster finalizes the split feature; completing the pre-existing legacy + // upload must still behave as before. + finalizeMpuPartsTableSplit(); + + OMRequest completeMultipartRequest = doPreExecuteCompleteMPU(volumeName, + bucketName, keyName, multipartUploadID, new ArrayList<>()); + + S3MultipartUploadCompleteRequest s3MultipartUploadCompleteRequest = + getS3MultipartUploadCompleteReq(completeMultipartRequest); + + OMClientResponse omClientResponse = + s3MultipartUploadCompleteRequest.validateAndUpdateCache(ozoneManager, + 3L); + + assertEquals(OzoneManagerProtocolProtos.Status.INVALID_REQUEST, + omClientResponse.getOMResponse().getStatus()); + } + protected void addVolumeAndBucket(String volumeName, String bucketName) throws Exception { OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java index 38c8708..6efa4f7 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java
@@ -17,6 +17,7 @@ package org.apache.hadoop.ozone.om.request.snapshot; +import static org.apache.hadoop.ozone.OzoneConsts.TRANSACTION_INFO_KEY; import static org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE; import static org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED; import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.createSnapshotRequest; @@ -25,6 +26,7 @@ import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.DeleteSnapshot; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -34,8 +36,10 @@ import java.util.UUID; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.ResolvedBucket; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -277,6 +281,51 @@ public void testEntryExist() throws Exception { assertEquals(1, getOmMetrics().getNumSnapshotDeleteFails()); } + /** + * Regression test for the flush-lag reclamation window. This is a companion to + * TestReclaimableKeyFilter#testKeyReclaimableWhenChainEmptyingPurgeUnflushedButDeleteFlushed. + * + * <p>Before OMSnapshotDeleteRequest stamped lastTransactionInfo, snapshot deletion updated only status and + * deletionTime. areSnapshotChangesFlushedToDB() therefore used the stale create-time stamp and reported an + * applied-but-unflushed deletion as flushed. SnapshotDeletingService#shouldIgnoreSnapshot relies on that method + * to defer processing until a snapshot's latest change is durable; the missing stamp allowed moveTableKeys and + * purge to be submitted before the double buffer flushed the deletion. + */ + @Test + public void testSnapshotDeleteIsNotReportedFlushedUntilFlushed() throws Exception { + when(getOzoneManager().isAdmin(any())).thenReturn(true); + String key = SnapshotInfo.getTableKey(getVolumeName(), getBucketName(), snapshotName); + + // Create the snapshot at transaction index 1; validateAndUpdateCache stamps lastTransactionInfo. + OMRequest createRequest = createSnapshotRequest(getVolumeName(), getBucketName(), snapshotName); + OMSnapshotCreateRequest omSnapshotCreateRequest = + TestOMSnapshotCreateRequest.doPreExecute(createRequest, getOzoneManager()); + omSnapshotCreateRequest.validateAndUpdateCache(getOzoneManager(), 1L); + SnapshotInfo snapshotInfo = getOmMetadataManager().getSnapshotInfoTable().get(key); + assertNotNull(snapshotInfo); + assertNotNull(snapshotInfo.getLastTransactionInfo(), "sanity: create stamps lastTransactionInfo"); + + // The double buffer flushes through the create transaction: persist the create's transaction info as + // the OM's flushed marker. Sanity: the snapshot's changes are now reported flushed. + getOmMetadataManager().getTransactionInfoTable().put(TRANSACTION_INFO_KEY, + TransactionInfo.fromByteString(snapshotInfo.getLastTransactionInfo())); + assertTrue(OmSnapshotManager.areSnapshotChangesFlushedToDB(getOmMetadataManager(), key), + "sanity: the create transaction is flushed"); + + // Delete the snapshot at transaction index 2. The change is applied to the table cache only; the + // double buffer has NOT flushed it (the flushed marker still points at the create transaction). + OMSnapshotDeleteRequest omSnapshotDeleteRequest = + doPreExecute(deleteSnapshotRequest(getVolumeName(), getBucketName(), snapshotName)); + omSnapshotDeleteRequest.validateAndUpdateCache(getOzoneManager(), 2L); + snapshotInfo = getOmMetadataManager().getSnapshotInfoTable().get(key); + assertEquals(SNAPSHOT_DELETED, snapshotInfo.getSnapshotStatus()); + + // The deletion (index 2) is not durable yet, so the snapshot's changes must not be reported as flushed. + // This verifies that lastTransactionInfo advanced from the create transaction to the delete transaction. + assertFalse(OmSnapshotManager.areSnapshotChangesFlushedToDB(getOmMetadataManager(), key), + "snapshot deletion at index 2 must remain unflushed while the marker is at the create transaction"); + } + private OMSnapshotDeleteRequest doPreExecute( OMRequest originalRequest) throws Exception { OMSnapshotDeleteRequest omSnapshotDeleteRequest =
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java index 6b8f2e6..89746fb 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java
@@ -23,6 +23,7 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS; +import static org.apache.hadoop.ozone.OzoneConsts.ETAG; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_DELETE_BATCH_SIZE; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_ENABLED; @@ -31,6 +32,7 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; import static org.apache.hadoop.ozone.om.OmConfig.Keys.ENABLE_FILESYSTEM_PATHS; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_REQUEST; import static org.apache.hadoop.ozone.om.helpers.BucketLayout.FILE_SYSTEM_OPTIMIZED; @@ -50,10 +52,12 @@ import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableMap; import java.io.File; @@ -77,6 +81,7 @@ import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; @@ -115,12 +120,16 @@ import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState; import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmMultipartPartInfo; +import org.apache.hadoop.ozone.om.helpers.OmMultipartPartKey; +import org.apache.hadoop.ozone.om.helpers.OmMultipartUpload; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.om.request.key.OMKeysDeleteRequest; +import org.apache.hadoop.ozone.om.request.util.OMMultipartUploadUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleConfiguration; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; @@ -146,6 +155,7 @@ import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.event.Level; @@ -3340,4 +3350,120 @@ private OmMultipartInfo createTestMultipartUpload(String volumeName, String buck public static String uniqueObjectName(String prefix) { return prefix + OBJECT_COUNTER.getAndIncrement(); } + + @Test + public void testPartCountIoExceptionSkipsUploadContinuesBucket(@TempDir File tempDir) + throws Exception { + OzoneConfiguration omConf = new OzoneConfiguration(); + omConf.set(OZONE_OM_DB_DIRS, tempDir.getAbsolutePath()); + OMMetadataManager realMetadataManager = new OmMetadataManagerImpl(omConf, null); + try { + String uploadA = OMMultipartUploadUtils.getMultipartUploadId(); + String uploadB = OMMultipartUploadUtils.getMultipartUploadId(); + String uploadC = OMMultipartUploadUtils.getMultipartUploadId(); + + addSplitSchemaPart(realMetadataManager, uploadA, 1); + addSplitSchemaPart(realMetadataManager, uploadA, 2); + addSplitSchemaPart(realMetadataManager, uploadC, 1); + + // Spy only the lightweight parts table and inject an IOException for + // uploadB's prefix scan to simulate a corrupt read; uploadA and uploadC + // fall through to the real table with real data. doThrow(...).when(...) + // is used (not when(...).thenThrow(...)) so the real iterator() is not + // invoked during stubbing, which would leak a native RocksDB iterator + // and block getStore().close(). + Table<OmMultipartPartKey, OmMultipartPartInfo> spyPartsTable = + Mockito.spy(realMetadataManager.getMultipartPartsTable()); + Mockito.doThrow(new RocksDatabaseException("simulated corruption")) + .when(spyPartsTable).iterator(eq(OmMultipartPartKey.prefix(uploadB))); + + OMMetadataManager mockMM = Mockito.mock(OMMetadataManager.class); + when(mockMM.getMultipartPartsTable()).thenReturn(spyPartsTable); + + assertEquals(2, OMMultipartUploadUtils.countParts(mockMM, uploadA)); + assertEquals(1, OMMultipartUploadUtils.countParts(mockMM, uploadC)); + assertThrows(IOException.class, + () -> OMMultipartUploadUtils.countParts(mockMM, uploadB)); + + KeyLifecycleService.PartCountLimitedList list = + new KeyLifecycleService.PartCountLimitedList(10); + for (String uploadId : Arrays.asList(uploadA, uploadB, uploadC)) { + try { + int partCount = OMMultipartUploadUtils.countParts(mockMM, uploadId); + list.add(new OmMultipartUpload("v", "b", "k", uploadId), partCount); + } catch (IOException e) { + // per-MPU skip — bucket loop continues + } + } + // A (2 parts) and C (1 part) are added; B is skipped due to IOException + assertEquals(2, list.size()); // uploadA and uploadC only + assertEquals(3, list.getPartCount()); // 2 (uploadA) + 1 (uploadC) + } finally { + realMetadataManager.getStore().close(); + } + } + + @Test + public void testPartCountLimitedListBoundaryBehavior() { + // Zero-parts upload is addable and does not fill the list + KeyLifecycleService.PartCountLimitedList list = + new KeyLifecycleService.PartCountLimitedList(5); + list.add(new OmMultipartUpload("v", "b", "k", "id1"), 0); + assertEquals(1, list.size()); + assertEquals(0, list.getPartCount()); + assertFalse(list.isFull()); + assertFalse(list.isEmpty()); + + // Exact boundary: partCount == maxPartCount triggers isFull + KeyLifecycleService.PartCountLimitedList exactList = + new KeyLifecycleService.PartCountLimitedList(5); + exactList.add(new OmMultipartUpload("v", "b", "k", "id2"), 5); + assertEquals(1, exactList.size()); + assertEquals(5, exactList.getPartCount()); + assertTrue(exactList.isFull()); + + // Over boundary: cumulative parts exceed max + KeyLifecycleService.PartCountLimitedList overList = + new KeyLifecycleService.PartCountLimitedList(5); + overList.add(new OmMultipartUpload("v", "b", "k", "id3"), 3); + assertFalse(overList.isFull()); + overList.add(new OmMultipartUpload("v", "b", "k", "id4"), 3); + assertTrue(overList.isFull()); + assertEquals(2, overList.size()); + assertEquals(6, overList.getPartCount()); + + // clear() resets all state + overList.clear(); + assertTrue(overList.isEmpty()); + assertFalse(overList.isFull()); + assertEquals(0, overList.size()); + assertEquals(0, overList.getPartCount()); + } + + private static void addSplitSchemaPart(OMMetadataManager omMetadataManager, + String uploadId, int partNumber) throws IOException { + OmKeyLocationInfo locationInfo = new OmKeyLocationInfo.Builder() + .setBlockID(new BlockID(1L, partNumber)) + .setLength(100) + .build(); + OmKeyLocationInfoGroup locationGroup = new OmKeyLocationInfoGroup(0, + Collections.singletonList(locationInfo)); + String partName = "part-" + partNumber; + OmKeyInfo keyInfo = new OmKeyInfo.Builder() + .setVolumeName("v") + .setBucketName("b") + .setKeyName("k") + .setReplicationConfig(RatisReplicationConfig.getInstance(THREE)) + .setDataSize(100L) + .setCreationTime(System.currentTimeMillis()) + .setModificationTime(System.currentTimeMillis()) + .setObjectID(partNumber) + .setUpdateID(partNumber) + .addOmKeyLocationInfoGroup(locationGroup) + .addMetadata(ETAG, "etag-" + partNumber) + .build(); + OmMultipartPartInfo partInfo = OmMultipartPartInfo.from(partName, partNumber, keyInfo); + omMetadataManager.getMultipartPartsTable().put( + OmMultipartPartKey.of(uploadId, partNumber), partInfo); + } }
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/filter/TestReclaimableKeyFilter.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/filter/TestReclaimableKeyFilter.java index 5e781dd..ed9252c 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/filter/TestReclaimableKeyFilter.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/filter/TestReclaimableKeyFilter.java
@@ -30,6 +30,7 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.om.KeyManager; +import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; @@ -40,6 +41,7 @@ import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -315,4 +317,73 @@ public void testExclusiveSizeCalculationWithNonReclaimableKey(int actualNumberOf testReclaimableKeyFilter(volume, bucket, index, keyInfo, prevKeyInfo, prevPrevKeyInfo1, prevKeyInfo == null, size, replicatedSize); } + + /** + * Boundary of the flush-lag reclamation window: the purge of the last path snapshot has been APPLIED + * (the in-memory chain is empty) but is NOT yet flushed, so the on-disk snapshotInfoTable still holds the + * snapshot's row. Reclamation proceeding here is safe ONLY because the row can never be ACTIVE on disk: + * SnapshotDeletingService submits a purge only after the snapshot's deletion is flushed + * (shouldIgnoreSnapshot -> areSnapshotChangesFlushedToDB, which relies on OMSnapshotDeleteRequest stamping + * lastTransactionInfo). A DELETED-on-disk row is not user-readable, and re-processing it after a + * restore-from-backup re-runs moveTableKeys/purge idempotently. If a chain-removal path that bypasses that + * gate is ever added, this assumption breaks and reclamation here would resurrect an ACTIVE snapshot with + * physically deleted blocks. + */ + @Test + public void testKeyReclaimableWhenChainEmptyingPurgeUnflushedButDeleteFlushed() + throws IOException, RocksDBException { + setup(2, 1, 1, 1, 1); + String volume = getVolumes().get(0); + String bucket = getBuckets().get(0); + SnapshotInfo purgedSnapshot = getSnapshotInfos().get(getKey(volume, bucket)).get(0); + purgedSnapshot.setSnapshotStatus(SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED); + + // SnapshotPurge has been applied: the in-memory chain no longer has the snapshot... + getSnapshotInfos().get(getKey(volume, bucket)).clear(); + + // ReclaimableKeyFilter determines that the chain is empty without consulting snapshotInfoTable. These stubs + // document the invariant guaranteed by the SnapshotDeletingService flush gate: after the applied purge removes + // the snapshot from the in-memory chain, the unflushed on-disk row can only be DELETED. + Table<String, SnapshotInfo> snapshotInfoTable = mock(Table.class); + OMMetadataManager metadataManager = getOzoneManager().getMetadataManager(); + when(metadataManager.getSnapshotInfoTable()).thenReturn(snapshotInfoTable); + when(snapshotInfoTable.get(eq(purgedSnapshot.getTableKey()))).thenReturn(null); + when(snapshotInfoTable.getSkipCache(eq(purgedSnapshot.getTableKey()))).thenReturn(purgedSnapshot); + + OmKeyInfo keyInfo = getMockedOmKeyInfo(1); + when(keyInfo.getVolumeName()).thenReturn(volume); + when(keyInfo.getBucketName()).thenReturn(bucket); + + assertTrue(getReclaimableFilter().apply(Table.newKeyValue("deletedKey", keyInfo)), + "with the chain empty and the unflushed purge's on-disk row at worst DELETED, the AOS deleted key " + + "is reclaimable"); + } + + /** + * Control for the flush-lag repro above: when the chain is empty AND durably so (the on-disk + * snapshotInfoTable has no row either), reclamation must proceed. + */ + @Test + public void testKeyReclaimableWhenChainDurablyEmpty() throws IOException, RocksDBException { + setup(2, 1, 1, 1, 1); + String volume = getVolumes().get(0); + String bucket = getBuckets().get(0); + SnapshotInfo purgedSnapshot = getSnapshotInfos().get(getKey(volume, bucket)).get(0); + + getSnapshotInfos().get(getKey(volume, bucket)).clear(); + + // As above, these stubs document the durable-empty invariant but do not affect the filter result. + Table<String, SnapshotInfo> snapshotInfoTable = mock(Table.class); + OMMetadataManager metadataManager = getOzoneManager().getMetadataManager(); + when(metadataManager.getSnapshotInfoTable()).thenReturn(snapshotInfoTable); + when(snapshotInfoTable.get(eq(purgedSnapshot.getTableKey()))).thenReturn(null); + when(snapshotInfoTable.getSkipCache(eq(purgedSnapshot.getTableKey()))).thenReturn(null); + + OmKeyInfo keyInfo = getMockedOmKeyInfo(1); + when(keyInfo.getVolumeName()).thenReturn(volume); + when(keyInfo.getBucketName()).thenReturn(bucket); + + assertTrue(getReclaimableFilter().apply(Table.newKeyValue("deletedKey", keyInfo)), + "with no snapshot in the chain and none on disk, the AOS deleted key is reclaimable"); + } }
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java index e640c1e..a9c2c8b 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java
@@ -171,8 +171,12 @@ public int read(long position, ByteBuffer buf) throws IOException { } if (inputStream instanceof ExtendedInputStream) { final int remainingBeforeRead = buf.remaining(); - if (((ExtendedInputStream) inputStream).readFully(position, buf)) { - return remainingBeforeRead - buf.remaining(); + try { + if (((ExtendedInputStream) inputStream).readFully(position, buf)) { + return remainingBeforeRead - buf.remaining(); + } + } catch (EOFException e) { + return -1; } }
diff --git a/hadoop-ozone/recon/pom.xml b/hadoop-ozone/recon/pom.xml index 84d466c..c2d25b2 100644 --- a/hadoop-ozone/recon/pom.xml +++ b/hadoop-ozone/recon/pom.xml
@@ -231,10 +231,6 @@ </dependency> <dependency> <groupId>org.glassfish.jersey.core</groupId> - <artifactId>jersey-common</artifactId> - </dependency> - <dependency> - <groupId>org.glassfish.jersey.core</groupId> <artifactId>jersey-server</artifactId> </dependency> <dependency>
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconRestServletModule.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconRestServletModule.java index 544e227..28a138a 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconRestServletModule.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconRestServletModule.java
@@ -23,6 +23,9 @@ import java.net.URL; import java.util.HashMap; import java.util.Map; +import javax.inject.Inject; +import javax.servlet.ServletContext; +import javax.ws.rs.core.Context; import javax.ws.rs.core.UriBuilder; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -31,10 +34,7 @@ import org.apache.hadoop.ozone.recon.api.filters.ReconAuthFilter; import org.apache.hadoop.ozone.recon.chatbot.ChatbotConfigKeys; import org.glassfish.hk2.api.ServiceLocator; -import org.glassfish.jersey.internal.inject.InjectionManager; import org.glassfish.jersey.server.ResourceConfig; -import org.glassfish.jersey.server.spi.Container; -import org.glassfish.jersey.server.spi.ContainerLifecycleListener; import org.glassfish.jersey.servlet.ServletContainer; import org.jvnet.hk2.guice.bridge.api.GuiceBridge; import org.jvnet.hk2.guice.bridge.api.GuiceIntoHK2Bridge; @@ -125,31 +125,14 @@ private void checkIfPackageExistsAndLog(String pkg) { * Class to bridge Guice bindings to Jersey hk2 bindings. */ class GuiceResourceConfig extends ResourceConfig { - GuiceResourceConfig() { - register(new ContainerLifecycleListener() { - - @Override - public void onStartup(Container container) { - ServletContainer servletContainer = (ServletContainer) container; - InjectionManager injectionManager = container.getApplicationHandler() - .getInjectionManager(); - ServiceLocator serviceLocator = injectionManager - .getInstance(ServiceLocator.class); - GuiceBridge.getGuiceBridge().initializeGuiceBridge(serviceLocator); - GuiceIntoHK2Bridge guiceBridge = serviceLocator - .getService(GuiceIntoHK2Bridge.class); - Injector injector = (Injector) servletContainer.getServletContext() - .getAttribute(Injector.class.getName()); - guiceBridge.bridgeGuiceInjector(injector); - } - - @Override - public void onReload(Container container) { - } - - @Override - public void onShutdown(Container container) { - } - }); + @Inject + GuiceResourceConfig(ServiceLocator serviceLocator, + @Context ServletContext servletContext) { + GuiceBridge.getGuiceBridge().initializeGuiceBridge(serviceLocator); + GuiceIntoHK2Bridge guiceBridge = serviceLocator + .getService(GuiceIntoHK2Bridge.class); + Injector injector = (Injector) servletContext + .getAttribute(Injector.class.getName()); + guiceBridge.bridgeGuiceInjector(injector); } }
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/TriggerDBSyncEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/TriggerDBSyncEndpoint.java index 93e0f0c..493f2bf 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/TriggerDBSyncEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/TriggerDBSyncEndpoint.java
@@ -25,6 +25,7 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; +import org.apache.hadoop.ozone.recon.api.types.OMDBReprocessResponse; import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; import org.apache.hadoop.ozone.recon.spi.OzoneManagerServiceProvider; @@ -55,6 +56,17 @@ public Response triggerOMDBSync() { } @POST + @Path("om/reinit") + public Response triggerOMDBReinit() { + OMDBReprocessResponse response = ozoneManagerServiceProvider.triggerTaskRebuild(); + if (response.getStatus() == OMDBReprocessResponse.Status.ACCEPTED) { + return Response.accepted(response).build(); + } else { + return Response.status(Response.Status.CONFLICT).entity(response).build(); + } + } + + @POST @Path("scm/snapshot") public Response triggerSCMDBSnapshotSync() { ReconStorageContainerManagerFacade.ScmDbSnapshotTriggerResponse response =
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/OMDBReprocessResponse.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/OMDBReprocessResponse.java new file mode 100644 index 0000000..3988803 --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/OMDBReprocessResponse.java
@@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api.types; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Response for OM DB manual reprocess request. + */ +public class OMDBReprocessResponse { + + @JsonProperty("status") + private Status status; + + @JsonProperty("message") + private String message; + + /** Result of a manual OM DB reprocess request. */ + public enum Status { + ACCEPTED, + RETRY + } + + public OMDBReprocessResponse(Status status, String message) { + this.status = status; + this.message = message; + } + + public Status getStatus() { + return status; + } + + public String getMessage() { + return message; + } +}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOmMetadataManagerImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOmMetadataManagerImpl.java index ae0059f..d0f92d0 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOmMetadataManagerImpl.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOmMetadataManagerImpl.java
@@ -104,6 +104,7 @@ public void start(OzoneConfiguration configuration) throws IOException { LOG.info("Starting ReconOMMetadataManagerImpl"); File reconDbDir = reconUtils.getReconDbDir(configuration, OZONE_RECON_OM_SNAPSHOT_DB_DIR); + LOG.info("reconDbDir is: {}", reconDbDir.getAbsolutePath()); File lastKnownOMSnapshot = reconUtils.getLastKnownDB(reconDbDir, RECON_OM_SNAPSHOT_DB); if (lastKnownOMSnapshot != null) {
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/OzoneManagerServiceProvider.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/OzoneManagerServiceProvider.java index dc32b16..1bc45c6 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/OzoneManagerServiceProvider.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/OzoneManagerServiceProvider.java
@@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.recon.spi; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.recon.api.types.OMDBReprocessResponse; /** * Interface to access OM endpoints. @@ -45,4 +46,10 @@ public interface OzoneManagerServiceProvider { * @return whether the trigger happened or not */ boolean triggerSyncDataFromOMImmediately(); + + /** + * Trigger the OM DB rebuild process. + * @return OMDBReprocessResponse containing the status of the request. + */ + OMDBReprocessResponse triggerTaskRebuild(); }
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java index bec4908..d221168 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java
@@ -85,6 +85,7 @@ import org.apache.hadoop.ozone.recon.ReconContext; import org.apache.hadoop.ozone.recon.ReconServerConfigKeys; import org.apache.hadoop.ozone.recon.ReconUtils; +import org.apache.hadoop.ozone.recon.api.types.OMDBReprocessResponse; import org.apache.hadoop.ozone.recon.metrics.OzoneManagerSyncMetrics; import org.apache.hadoop.ozone.recon.metrics.ReconSyncMetrics; import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; @@ -562,6 +563,27 @@ boolean updateReconOmDBWithNewSnapshot() throws IOException { } } + @Override + public OMDBReprocessResponse triggerTaskRebuild() { + if (omMetadataManager == null || omMetadataManager.getStore() == null) { + return new OMDBReprocessResponse(OMDBReprocessResponse.Status.RETRY, + "Recon has not loaded an OM DB yet, so there is nothing to rebuild. Ensure an OM DB snapshot is " + + "present in the Recon OM DB directory and has been loaded, then retry."); + } + + ReconTaskController.ReInitializationResult result = reconTaskController.queueReInitializationEvent( + ReconTaskReInitializationEvent.ReInitializationReason.MANUAL_OM_DB_REBUILD); + + if (result == ReconTaskController.ReInitializationResult.SUCCESS) { + return new OMDBReprocessResponse(OMDBReprocessResponse.Status.ACCEPTED, + "Manual OM DB rebuild queued successfully."); + } else { + return new OMDBReprocessResponse(OMDBReprocessResponse.Status.RETRY, + "Manual OM DB rebuild could not be queued. Buffer might be full or another rebuild is " + + "pending. Please retry."); + } + } + /** * Get Delta updates from OM through RPC call and apply to local OM DB as * well as accumulate in a buffer.
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java index 10d1838..dc037e4 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java
@@ -22,7 +22,6 @@ import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBDefinition.CONTAINER_KEY_COUNT; import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBDefinition.KEY_CONTAINER; import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBDefinition.REPLICA_HISTORY_V2; -import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBProvider.truncateTable; import jakarta.annotation.Nonnull; import java.io.IOException; @@ -127,9 +126,15 @@ public void reinitWithNewContainerDataFromOm(Map<ContainerKeyPrefix, Integer> containerKeyPrefixCounts) throws IOException { // clear and re-init all container-related tables - truncateTable(this.containerKeyTable); - truncateTable(this.keyContainerTable); - truncateTable(this.containerKeyCountTable); + if (containerKeyTable != null) { + containerKeyTable.clear(); + } + if (keyContainerTable != null) { + keyContainerTable.clear(); + } + if (containerKeyCountTable != null) { + containerKeyCountTable.clear(); + } initializeTables(); if (containerKeyPrefixCounts != null) {
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBProvider.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBProvider.java index 16a303a..9dd6874 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBProvider.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBProvider.java
@@ -31,9 +31,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; -import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.hdds.utils.db.Table.KeyValue; -import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.recon.ReconUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -111,18 +108,6 @@ public DBStore getDbStore() { return dbStore; } - static void truncateTable(Table table) throws IOException { - if (table == null) { - return; - } - try (TableIterator<Object, Table.KeyValue<Object, Object>> tableIterator = table.iterator()) { - while (tableIterator.hasNext()) { - KeyValue<Object, Object> entry = tableIterator.next(); - table.delete(entry.getKey()); - } - } - } - private static DBStore initializeDBStore(OzoneConfiguration configuration, String dbName) { DBStore dbStore = null;
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconFileMetadataManagerImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconFileMetadataManagerImpl.java index 3a1d2b7..25b60f6 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconFileMetadataManagerImpl.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconFileMetadataManagerImpl.java
@@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.recon.spi.impl; import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBDefinition.FILE_COUNT_BY_SIZE; -import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBProvider.truncateTable; import java.io.IOException; import javax.inject.Inject; @@ -108,7 +107,9 @@ public void commitBatchOperation(RDBBatchOperation rdbBatchOperation) @Override public void clearFileCountTable() throws IOException { - truncateTable(fileCountTable); + if (fileCountTable != null) { + fileCountTable.clear(); + } LOG.info("Successfully cleared file count table"); } }
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNamespaceSummaryManagerImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNamespaceSummaryManagerImpl.java index 1d0a7a0..0287859 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNamespaceSummaryManagerImpl.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNamespaceSummaryManagerImpl.java
@@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.recon.spi.impl; import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBDefinition.NAMESPACE_SUMMARY; -import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBProvider.truncateTable; import java.io.IOException; import javax.inject.Inject; @@ -66,7 +65,7 @@ public void reinitialize(ReconDBProvider reconDBProvider) throws IOException { @Override public void clearNSSummaryTable() throws IOException { - truncateTable(nsSummaryTable); + nsSummaryTable.clear(); } @Override
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/MultipartInfoInsightHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/MultipartInfoInsightHandler.java index 4f14de2..e4439c5 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/MultipartInfoInsightHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/MultipartInfoInsightHandler.java
@@ -18,12 +18,19 @@ package org.apache.hadoop.ozone.recon.tasks; import java.io.IOException; +import java.util.HashMap; import java.util.Map; +import org.apache.commons.lang3.tuple.MutablePair; import org.apache.commons.lang3.tuple.Triple; +import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmMultipartPartInfo; +import org.apache.hadoop.ozone.om.helpers.OmMultipartPartKey; +import org.apache.hadoop.ozone.om.helpers.QuotaUtil; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo; import org.apache.hadoop.ozone.recon.api.types.ReconBasicOmKeyInfo; import org.slf4j.Logger; @@ -32,6 +39,25 @@ /** * Manages records in the MultipartInfo Table, updating counts and sizes of * multipart upload keys in the backend. + * + * <p>Multipart uploads are stored in one of two schemas: + * <ul> + * <li>Legacy (schemaVersion {@link OmMultipartKeyInfo#LEGACY_SCHEMA_VERSION}): + * the part information is embedded inside the {@code multipartInfoTable} value + * (see {@link OmMultipartKeyInfo#getPartKeyInfoMap()}).</li> + * <li>Split parts-table (schemaVersion + * {@link OmMultipartKeyInfo#SPLIT_PARTS_TABLE_SCHEMA_VERSION}): the + * {@code multipartInfoTable} value carries no embedded parts; each part is a + * separate row in the {@code multipartPartsTable}, keyed by + * {@code uploadId/partNumber}.</li> + * </ul> + * + * <p>The event handlers below only access {@code multipartInfoTable} events, + * whose values embed parts only for the legacy schema. Split-schema part + * sizes therefore cannot be accounted incrementally from these events (the + * handler has no DB access and the event carries no part data). + * So they are reconciled during the periodic reprocess in {@link #getTableSizeAndCount(String, OMMetadataManager)}, + * which reads the split {@code multipartPartsTable} directly. */ public class MultipartInfoInsightHandler implements OmTableHandler { @@ -39,6 +65,14 @@ public class MultipartInfoInsightHandler implements OmTableHandler { LoggerFactory.getLogger(MultipartInfoInsightHandler.class); /** + * Consumes the (unreplicated, replicated) size of a single multipart part. + */ + @FunctionalInterface + private interface PartSizeConsumer { + void accept(long dataSize, long replicatedSize); + } + + /** * Invoked by the process method to add information on those keys that have * been initiated for multipart upload in the backend. */ @@ -50,14 +84,7 @@ public void handlePutEvent(OMDBUpdateEvent<String, Object> event, String tableNa OmMultipartKeyInfo multipartKeyInfo = (OmMultipartKeyInfo) event.getValue(); objectCountMap.computeIfPresent(getTableCountKeyFromTable(tableName), (k, count) -> count + 1L); - - for (PartKeyInfo partKeyInfo : multipartKeyInfo.getPartKeyInfoMap()) { - ReconBasicOmKeyInfo omKeyInfo = ReconBasicOmKeyInfo.getFromProtobuf(partKeyInfo.getPartKeyInfo()); - unReplicatedSizeMap.computeIfPresent(getUnReplicatedSizeKeyFromTable(tableName), - (k, size) -> size + omKeyInfo.getDataSize()); - replicatedSizeMap.computeIfPresent(getReplicatedSizeKeyFromTable(tableName), - (k, size) -> size + omKeyInfo.getReplicatedSize()); - } + applyLegacyPartSizes(multipartKeyInfo, tableName, unReplicatedSizeMap, replicatedSizeMap, true); } else { LOG.warn("Put event does not have the Multipart Key Info for {}.", event.getKey()); } @@ -75,28 +102,7 @@ public void handleDeleteEvent(OMDBUpdateEvent<String, Object> event, String tabl OmMultipartKeyInfo multipartKeyInfo = (OmMultipartKeyInfo) event.getValue(); objectCountMap.computeIfPresent(getTableCountKeyFromTable(tableName), (k, count) -> count > 0 ? count - 1L : 0L); - - for (PartKeyInfo partKeyInfo : multipartKeyInfo.getPartKeyInfoMap()) { - ReconBasicOmKeyInfo omKeyInfo = ReconBasicOmKeyInfo.getFromProtobuf(partKeyInfo.getPartKeyInfo()); - unReplicatedSizeMap.computeIfPresent(getUnReplicatedSizeKeyFromTable(tableName), - (k, size) -> { - long newSize = size > omKeyInfo.getDataSize() ? size - omKeyInfo.getDataSize() : 0L; - if (newSize < 0) { - LOG.warn("Negative unreplicated size for key: {}. Original: {}, Part: {}", - k, size, omKeyInfo.getDataSize()); - } - return newSize; - }); - replicatedSizeMap.computeIfPresent(getReplicatedSizeKeyFromTable(tableName), - (k, size) -> { - long newSize = size > omKeyInfo.getReplicatedSize() ? size - omKeyInfo.getReplicatedSize() : 0L; - if (newSize < 0) { - LOG.warn("Negative replicated size for key: {}. Original: {}, Part: {}", - k, size, omKeyInfo.getReplicatedSize()); - } - return newSize; - }); - } + applyLegacyPartSizes(multipartKeyInfo, tableName, unReplicatedSizeMap, replicatedSizeMap, false); } else { LOG.warn("Delete event does not have the Multipart Key Info for {}.", event.getKey()); } @@ -116,28 +122,13 @@ public void handleUpdateEvent(OMDBUpdateEvent<String, Object> event, String tabl return; } - // In Update event the count for the multipart info table will not change. So we - // don't need to update the count. + // In an Update event the count for the multipart info table does not + // change, so only the sizes are adjusted: subtract the old parts and add + // the new parts. OmMultipartKeyInfo oldMultipartKeyInfo = (OmMultipartKeyInfo) event.getOldValue(); OmMultipartKeyInfo newMultipartKeyInfo = (OmMultipartKeyInfo) event.getValue(); - - // Calculate old sizes - for (PartKeyInfo partKeyInfo : oldMultipartKeyInfo.getPartKeyInfoMap()) { - ReconBasicOmKeyInfo omKeyInfo = ReconBasicOmKeyInfo.getFromProtobuf(partKeyInfo.getPartKeyInfo()); - unReplicatedSizeMap.computeIfPresent(getUnReplicatedSizeKeyFromTable(tableName), - (k, size) -> size - omKeyInfo.getDataSize()); - replicatedSizeMap.computeIfPresent(getReplicatedSizeKeyFromTable(tableName), - (k, size) -> size - omKeyInfo.getReplicatedSize()); - } - - // Calculate new sizes - for (PartKeyInfo partKeyInfo : newMultipartKeyInfo.getPartKeyInfoMap()) { - ReconBasicOmKeyInfo omKeyInfo = ReconBasicOmKeyInfo.getFromProtobuf(partKeyInfo.getPartKeyInfo()); - unReplicatedSizeMap.computeIfPresent(getUnReplicatedSizeKeyFromTable(tableName), - (k, size) -> size + omKeyInfo.getDataSize()); - replicatedSizeMap.computeIfPresent(getReplicatedSizeKeyFromTable(tableName), - (k, size) -> size + omKeyInfo.getReplicatedSize()); - } + applyLegacyPartSizes(oldMultipartKeyInfo, tableName, unReplicatedSizeMap, replicatedSizeMap, false); + applyLegacyPartSizes(newMultipartKeyInfo, tableName, unReplicatedSizeMap, replicatedSizeMap, true); } else { LOG.warn("Update event does not have the Multipart Key Info for {}.", event.getKey()); } @@ -148,13 +139,27 @@ public void handleUpdateEvent(OMDBUpdateEvent<String, Object> event, String tabl * counts for the multipart info table. Additionally, it computes the sizes * of both replicated and unreplicated parts that are currently in multipart * uploads in the backend. + * + * <p>This is schema-aware: legacy (schemaVersion 0) part sizes are read from + * the embedded {@link OmMultipartKeyInfo#getPartKeyInfoMap()}, while split + * (schemaVersion 1) part sizes are summed from the separate + * {@code multipartPartsTable}. The count returned is always the number of + * multipart uploads (rows in the {@code multipartInfoTable}), regardless of + * schema. */ @Override public Triple<Long, Long, Long> getTableSizeAndCount(String tableName, OMMetadataManager omMetadataManager) throws IOException { long count = 0; - long unReplicatedSize = 0; - long replicatedSize = 0; + // left = unreplicated size, right = replicated size. A mutable pair is used + // so the running totals can be updated from the part-iteration lambda below. + final MutablePair<Long, Long> sizes = MutablePair.of(0L, 0L); + + // uploadId -> parent replication config, for split-schema MPUs. Their parts + // live in multipartPartsTable but do not carry a replication config, so the + // parent's config (recorded here) is used to compute their replicated size + // in the second pass below. + Map<String, ReplicationConfig> splitSchemaUploads = new HashMap<>(); Table<String, OmMultipartKeyInfo> table = (Table<String, OmMultipartKeyInfo>) omMetadataManager.getTable(tableName); @@ -163,15 +168,113 @@ public Triple<Long, Long, Long> getTableSizeAndCount(String tableName, Table.KeyValue<String, OmMultipartKeyInfo> kv = iterator.next(); if (kv != null && kv.getValue() != null) { OmMultipartKeyInfo multipartKeyInfo = kv.getValue(); - for (PartKeyInfo partKeyInfo : multipartKeyInfo.getPartKeyInfoMap()) { - ReconBasicOmKeyInfo omKeyInfo = ReconBasicOmKeyInfo.getFromProtobuf(partKeyInfo.getPartKeyInfo()); - unReplicatedSize += omKeyInfo.getDataSize(); - replicatedSize += omKeyInfo.getReplicatedSize(); + if (isLegacySchema(multipartKeyInfo)) { + forEachLegacyPart(multipartKeyInfo, (dataSize, replicatedSize) -> { + sizes.setLeft(sizes.getLeft() + dataSize); + sizes.setRight(sizes.getRight() + replicatedSize); + }); + } else { + // Split schema: parts are stored in multipartPartsTable. Remember + // the parent replication config keyed by uploadId (last component + // of the multipart key) for the second pass. + splitSchemaUploads.put(getUploadIdFromMultipartKey(kv.getKey()), + multipartKeyInfo.getReplicationConfig()); } count++; } } } - return Triple.of(count, unReplicatedSize, replicatedSize); + + // Second pass: sum the sizes of split-schema parts from multipartPartsTable. + if (!splitSchemaUploads.isEmpty()) { + Table<OmMultipartPartKey, OmMultipartPartInfo> partsTable = + omMetadataManager.getMultipartPartsTable(); + try (TableIterator<OmMultipartPartKey, Table.KeyValue<OmMultipartPartKey, OmMultipartPartInfo>> partIterator = + partsTable.iterator()) { + while (partIterator.hasNext()) { + Table.KeyValue<OmMultipartPartKey, OmMultipartPartInfo> kv = partIterator.next(); + if (kv != null && kv.getKey() != null && kv.getValue() != null) { + ReplicationConfig replicationConfig = splitSchemaUploads.get(kv.getKey().getUploadId()); + if (replicationConfig == null) { + // Part whose parent MPU is not in multipartInfoTable (e.g. an + // orphan mid-cleanup). Skip to avoid mis-attributing its size. + continue; + } + long partDataSize = kv.getValue().getDataSize(); + sizes.setLeft(sizes.getLeft() + partDataSize); + sizes.setRight(sizes.getRight() + + QuotaUtil.getReplicatedSize(partDataSize, replicationConfig)); + } + } + } + } + + return Triple.of(count, sizes.getLeft(), sizes.getRight()); + } + + /** + * Adds (or subtracts) the sizes of a legacy MPU's embedded parts to the size + * maps. Split-schema MPUs carry no embedded parts and are ignored here (their + * sizes are reconciled during reprocess; see class Javadoc). + * + * @param add {@code true} to add the part sizes (PUT / new value in UPDATE), + * {@code false} to subtract them (DELETE / old value in UPDATE). + */ + private void applyLegacyPartSizes(OmMultipartKeyInfo multipartKeyInfo, String tableName, + Map<String, Long> unReplicatedSizeMap, Map<String, Long> replicatedSizeMap, boolean add) { + forEachLegacyPart(multipartKeyInfo, (dataSize, replicatedSize) -> { + updateSize(unReplicatedSizeMap, getUnReplicatedSizeKeyFromTable(tableName), dataSize, add, "unreplicated"); + updateSize(replicatedSizeMap, getReplicatedSizeKeyFromTable(tableName), replicatedSize, add, "replicated"); + }); + } + + /** + * Invokes {@code consumer} with the (unreplicated, replicated) size of each + * embedded part of a legacy MPU. Does nothing for split-schema MPUs. + */ + private static void forEachLegacyPart(OmMultipartKeyInfo multipartKeyInfo, PartSizeConsumer consumer) { + if (!isLegacySchema(multipartKeyInfo)) { + return; + } + for (PartKeyInfo partKeyInfo : multipartKeyInfo.getPartKeyInfoMap()) { + ReconBasicOmKeyInfo omKeyInfo = ReconBasicOmKeyInfo.getFromProtobuf(partKeyInfo.getPartKeyInfo()); + consumer.accept(omKeyInfo.getDataSize(), omKeyInfo.getReplicatedSize()); + } + } + + /** + * Adds or subtracts {@code delta} from the value stored under {@code key} in + * {@code sizeMap} (only if the key is already present). Subtraction is clamped + * at zero, and an underflow is logged (as it indicates an accounting anomaly, + * e.g. a delete/update for a part whose size was never added). + * + * @param sizeType a human-readable label ("unreplicated"/"replicated") used + * only in the underflow warning message. + */ + private static void updateSize(Map<String, Long> sizeMap, String key, long delta, boolean add, String sizeType) { + sizeMap.computeIfPresent(key, (k, size) -> { + if (add) { + return size + delta; + } + if (size < delta) { + LOG.warn("Negative {} size for key: {}. Current: {}, Part: {}. Clamping to 0.", + sizeType, k, size, delta); + return 0L; + } + return size - delta; + }); + } + + private static boolean isLegacySchema(OmMultipartKeyInfo multipartKeyInfo) { + return multipartKeyInfo.getSchemaVersion() == OmMultipartKeyInfo.LEGACY_SCHEMA_VERSION; + } + + /** + * The multipart key is {@code .../uploadId}; the split parts-table rows are + * keyed by that same uploadId. Extract it from the last path component. + */ + private static String getUploadIdFromMultipartKey(String multipartKey) { + int idx = multipartKey.lastIndexOf(OzoneConsts.OM_KEY_PREFIX); + return idx >= 0 ? multipartKey.substring(idx + OzoneConsts.OM_KEY_PREFIX.length()) : multipartKey; } }
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ReconTaskControllerImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ReconTaskControllerImpl.java index 223d758..5b142c0 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ReconTaskControllerImpl.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ReconTaskControllerImpl.java
@@ -617,6 +617,10 @@ public synchronized ReconTaskController.ReInitializationResult queueReInitializa // Track reprocess submission controllerMetrics.incrTotalReprocessSubmittedToQueue(); + if (reason == ReconTaskReInitializationEvent.ReInitializationReason.MANUAL_OM_DB_REBUILD) { + lastRetryTimestamp.set(0); + } + ReInitializationResult reInitializationResult = validateRetryCountAndDelay(); if (null != reInitializationResult) { return reInitializationResult; @@ -746,9 +750,14 @@ public ReconOMMetadataManager createOMCheckpoint(ReconOMMetadataManager omMetaMa // Create temporary directory for checkpoint String parentPath = cleanTempCheckPointPath(omMetaManager); - // Create checkpoint + // Create checkpoint. getCheckpoint returns null when RocksDB fails to snapshot + // (e.g. a manually placed OM DB that is incomplete or corrupt). DBCheckpoint checkpoint = omMetaManager.getStore().getCheckpoint(parentPath, true); - + if (checkpoint == null) { + throw new IOException("Failed to create OM DB checkpoint at " + parentPath + + "; the on-disk OM DB may be incomplete or corrupt."); + } + return omMetaManager.createCheckpointReconMetadataManager(configuration, checkpoint); } @@ -840,7 +849,7 @@ public long getDroppedBatches() { * Reset retry counters - for testing purposes. */ @VisibleForTesting - void resetRetryCounters() { + public void resetRetryCounters() { eventProcessRetryCount.set(0); lastRetryTimestamp.set(0); } @@ -868,8 +877,8 @@ AtomicBoolean getTasksFailedFlag() { */ private void cleanupPreExistingCheckpoints() { try { - if (currentOMMetadataManager == null) { - LOG.debug("No current OM metadata manager, skipping pre-existing checkpoint cleanup"); + if (currentOMMetadataManager == null || currentOMMetadataManager.getStore() == null) { + LOG.debug("No current OM metadata manager or store, skipping pre-existing checkpoint cleanup"); return; }
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ReconTaskReInitializationEvent.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ReconTaskReInitializationEvent.java index e241c48..be89512 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ReconTaskReInitializationEvent.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ReconTaskReInitializationEvent.java
@@ -36,7 +36,8 @@ public class ReconTaskReInitializationEvent implements ReconEvent { public enum ReInitializationReason { BUFFER_OVERFLOW, TASK_FAILURES, - MANUAL_TRIGGER + MANUAL_TRIGGER, + MANUAL_OM_DB_REBUILD } public ReconTaskReInitializationEvent(ReInitializationReason reason,
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/capacity/Capacity.test.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/capacity/Capacity.test.tsx index 037fcd0..0f32046 100644 --- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/capacity/Capacity.test.tsx +++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/capacity/Capacity.test.tsx
@@ -25,7 +25,13 @@ import * as mockResponses from '@tests/mocks/capacityMocks/capacityResponseMocks'; vi.mock('@/components/autoReloadPanel/autoReloadPanel', () => ({ - default: () => <div data-testid="auto-reload-panel" />, + default: (props: { onReload?: () => void }) => ( + <div data-testid="auto-reload-panel"> + <button data-testid="manual-reload" onClick={() => props.onReload?.()}> + reload + </button> + </div> + ), })); vi.mock('@/components/eChart/eChart', () => ({ EChart: () => <div data-testid="echart" />, @@ -49,9 +55,9 @@ return; } await waitFor(() => - expect(ozoneCapacityCard).toHaveTextContent(/TOTAL\s*10\s*KB/i) + expect(ozoneCapacityCard).toHaveTextContent(/TOTAL CAPACITY\s*10\s*KB/i) ); - expect(ozoneCapacityCard).toHaveTextContent(/OZONE USED SPACE\s*4\s*KB/i); + expect(ozoneCapacityCard).toHaveTextContent(/USED SPACE\s*4\s*KB/i); expect(ozoneCapacityCard).toHaveTextContent(/OTHER USED SPACE\s*2\s*KB/i); expect(ozoneCapacityCard).toHaveTextContent(/CONTAINER PRE-ALLOCATED\s*1\s*KB/i); expect(ozoneCapacityCard).toHaveTextContent(/REMAINING SPACE\s*4\s*KB/i); @@ -63,7 +69,7 @@ return; } await waitFor(() => - expect(ozoneUsedSpaceCard).toHaveTextContent(/PENDING DELETION\s*6\s*KB/i) + expect(ozoneUsedSpaceCard).toHaveTextContent(/PENDING DELETION\s*7\s*KB/i) ); }); @@ -80,7 +86,7 @@ expect(pendingDeletionCard).toHaveTextContent(/OZONE MANAGER\s*2\s*KB/i) ); expect(pendingDeletionCard) - .toHaveTextContent(/STORAGE CONTAINER MANAGER\s*1\s*KB/i); + .toHaveTextContent(/STORAGE CONTAINER MANAGER\s*2\s*KB/i); expect(pendingDeletionCard).toHaveTextContent(/DATANODES\s*3\s*KB/i); const downloadLink = await screen.findByText('Download Insights'); @@ -95,7 +101,7 @@ expect(datanodeCard).toHaveTextContent(/FREE SPACE\s*3\s*KB/i); }); - test('clamps pendingBlockSize to 0 when selected datanode reports -1 (offline/unreachable)', async () => { + test('defaults to the first available datanode when the first datanode reports -1 (offline/unreachable)', async () => { capacityServer.use( rest.get('api/v1/pendingDeletion', (req, res, ctx) => { const component = req.url.searchParams.get('component'); @@ -130,13 +136,70 @@ if (!datanodeCard) { return; } - // dn-1 is selected by default; its pendingBlockSize is -1 (offline sentinel). - // PENDING DELETION should show 0 B, not a negative value. + // dn-1 is unavailable (pendingBlockSize -1), but dn-2 is healthy, so the page should + // default to dn-2 and show its capacity instead of landing on the error card. + // USED SPACE = used (2048) + pendingBlockSize (2048) = 4 KB, FREE SPACE = remaining + // (2048) + committed (1024) = 3 KB. await waitFor(() => - expect(datanodeCard).toHaveTextContent(/PENDING DELETION\s*0\s*B/i) + expect(datanodeCard).toHaveTextContent(/USED SPACE\s*4\s*KB/i) ); - // USED SPACE = used (4096) + clamped pendingBlockSize (0) = 4 KB - expect(datanodeCard).toHaveTextContent(/USED SPACE\s*4\s*KB/i); + expect(datanodeCard).toHaveTextContent(/FREE SPACE\s*3\s*KB/i); + expect(screen.queryByTestId('dn-used-space-error')).not.toBeInTheDocument(); + expect(screen.queryByTestId('dn-free-space-error')).not.toBeInTheDocument(); + // The dropdown label must reflect the actually-selected DN (dn-2), not the first + // (unavailable) option. + const selectionItem = datanodeCard.querySelector('.ant-select-selection-item'); + expect(selectionItem).toHaveTextContent('dn-2'); + expect(selectionItem).not.toHaveTextContent('dn-1'); + }); + + test('shows error card instead of outdated data when every datanode reports -1 (offline/unreachable)', async () => { + capacityServer.use( + rest.get('api/v1/pendingDeletion', (req, res, ctx) => { + const component = req.url.searchParams.get('component'); + if (component === 'dn') { + return res( + ctx.status(200), + ctx.json({ + ...mockResponses.DnPendingDeletion, + pendingDeletionPerDataNode: [ + { hostName: 'dn-1', datanodeUuid: 'uuid-1', pendingBlockSize: -1 }, + { hostName: 'dn-2', datanodeUuid: 'uuid-2', pendingBlockSize: -1 } + ] + }) + ); + } + const map: Record<string, object> = { + scm: mockResponses.ScmPendingDeletion, + om: mockResponses.OmPendingDeletion + }; + const body = component ? map[component] : undefined; + return body + ? res(ctx.status(200), ctx.json(body)) + : res(ctx.status(400), ctx.json({ message: 'Unsupported pending deletion component.' })); + }) + ); + + render(<Capacity />); + + const downloadLink = await screen.findByText('Download Insights'); + const datanodeCard = downloadLink.closest('.ant-card'); + expect(datanodeCard).not.toBeNull(); + if (!datanodeCard) { + return; + } + // No DN is available, so we fall back to dn-1; its pendingBlockSize is -1 (offline + // sentinel), so the datanode is unavailable and its capacity data may be outdated. + // Show an error card for USED SPACE and FREE SPACE instead of the stale + // storage-report values. + expect(await screen.findByTestId('dn-used-space-error')).toBeInTheDocument(); + expect(await screen.findByTestId('dn-free-space-error')).toBeInTheDocument(); + await waitFor(() => + expect(datanodeCard).toHaveTextContent(/USED SPACE\s*N\/A/i) + ); + expect(datanodeCard).toHaveTextContent(/FREE SPACE\s*N\/A/i); + // With every DN unavailable, the dropdown falls back to the first DN (dn-1). + expect(datanodeCard.querySelector('.ant-select-selection-item')).toHaveTextContent('dn-1'); }); test('shows scm-only error state when SCM pending deletion returns sentinel failure values', async () => { @@ -307,4 +370,132 @@ ); expect(screen.queryByRole('option', { name: 'dn-17' })).not.toBeInTheDocument(); }); + + // Interval constants mirrored from capacity.tsx / autoReload.constants. + const PENDING_POLL_INTERVAL = 5 * 1000; + const AUTO_RELOAD_INTERVAL = 60 * 1000; + + type EndpointCounts = { storage: number; scm: number; om: number; dn: number }; + + // Installs handlers that count how often each endpoint is hit and serves the + // supplied DN scan statuses in order (clamped to the last entry afterwards). + const setupCountingHandlers = (dnStatuses: string[]) => { + const counts: EndpointCounts = { storage: 0, scm: 0, om: 0, dn: 0 }; + capacityServer.use( + rest.get('api/v1/storageDistribution', (_req, res, ctx) => { + counts.storage++; + return res(ctx.status(200), ctx.json(mockResponses.StorageDistribution)); + }), + rest.get('api/v1/pendingDeletion', (req, res, ctx) => { + const component = req.url.searchParams.get('component'); + if (component === 'dn') { + const status = dnStatuses[Math.min(counts.dn, dnStatuses.length - 1)]; + counts.dn++; + return res( + ctx.status(200), + ctx.json({ ...mockResponses.DnPendingDeletion, status }) + ); + } + if (component === 'scm') { + counts.scm++; + return res(ctx.status(200), ctx.json(mockResponses.ScmPendingDeletion)); + } + if (component === 'om') { + counts.om++; + return res(ctx.status(200), ctx.json(mockResponses.OmPendingDeletion)); + } + return res(ctx.status(400), ctx.json({ message: 'Unsupported pending deletion component.' })); + }) + ); + return counts; + }; + + test('Auto Refresh off: a manual refresh drives the DN scan, polling only the DN endpoint until it finishes, then syncs the other endpoints', async () => { + // Auto Refresh disabled before mount, so useAutoReload never starts its timer. + sessionStorage.setItem('autoReloadEnabled', 'false'); + + // Mount read is FINISHED (no scan running). The manual refresh kicks off a + // scan: the next reads are IN_PROGRESS, IN_PROGRESS, then FINISHED. + const counts = setupCountingHandlers([ + 'FINISHED', // mount + 'IN_PROGRESS', // manual refresh + 'IN_PROGRESS', // poll #1 + 'FINISHED' // poll #2 + ]); + + vi.useFakeTimers(); + try { + render(<Capacity />); + + // Flush the initial mount fetch: everything fetched exactly once. + await vi.advanceTimersByTimeAsync(50); + expect(counts).toEqual({ storage: 1, scm: 1, om: 1, dn: 1 }); + + // Scan is FINISHED, so nothing is polled while idle. + await vi.advanceTimersByTimeAsync(AUTO_RELOAD_INTERVAL * 2); + expect(counts).toEqual({ storage: 1, scm: 1, om: 1, dn: 1 }); + + // Manual reload -> full refresh of all four endpoints; DN comes back + // IN_PROGRESS, which starts the DN-only poll. + fireEvent.click(screen.getByTestId('manual-reload')); + await vi.advanceTimersByTimeAsync(50); + expect(counts).toEqual({ storage: 2, scm: 2, om: 2, dn: 2 }); + + // While IN_PROGRESS only the DN endpoint is polled; the others stay put. + await vi.advanceTimersByTimeAsync(PENDING_POLL_INTERVAL); + expect(counts).toEqual({ storage: 2, scm: 2, om: 2, dn: 3 }); + + // Next poll returns FINISHED -> DN polling stops and the other three + // endpoints are synced exactly once. + await vi.advanceTimersByTimeAsync(PENDING_POLL_INTERVAL); + expect(counts).toEqual({ storage: 3, scm: 3, om: 3, dn: 4 }); + + // No further polling once finished, and no periodic refresh while off. + await vi.advanceTimersByTimeAsync(AUTO_RELOAD_INTERVAL * 2); + expect(counts).toEqual({ storage: 3, scm: 3, om: 3, dn: 4 }); + } finally { + vi.useRealTimers(); + sessionStorage.removeItem('autoReloadEnabled'); + } + }); + + test('Auto Refresh on: refreshes all endpoints on the interval, polls only the DN endpoint while a scan runs, and syncs the others when it finishes', async () => { + // Auto Refresh enabled (default). useAutoReload runs a full refresh every 60s. + sessionStorage.setItem('autoReloadEnabled', 'true'); + + // A scan is already running at mount: IN_PROGRESS, IN_PROGRESS, then FINISHED. + const counts = setupCountingHandlers([ + 'IN_PROGRESS', // mount + 'IN_PROGRESS', // poll #1 + 'FINISHED' // poll #2 + ]); + + vi.useFakeTimers(); + try { + render(<Capacity />); + + // Flush the initial mount fetch: everything fetched once, scan IN_PROGRESS. + await vi.advanceTimersByTimeAsync(50); + expect(counts).toEqual({ storage: 1, scm: 1, om: 1, dn: 1 }); + + // While IN_PROGRESS only the DN endpoint is polled. + await vi.advanceTimersByTimeAsync(PENDING_POLL_INTERVAL); + expect(counts).toEqual({ storage: 1, scm: 1, om: 1, dn: 2 }); + + // Next poll returns FINISHED -> DN polling stops and the others sync once. + await vi.advanceTimersByTimeAsync(PENDING_POLL_INTERVAL); + expect(counts).toEqual({ storage: 2, scm: 2, om: 2, dn: 3 }); + + // No DN-only polling once finished. + await vi.advanceTimersByTimeAsync(PENDING_POLL_INTERVAL * 4); + expect(counts).toEqual({ storage: 2, scm: 2, om: 2, dn: 3 }); + + // At the auto-refresh interval, all four endpoints are refreshed together. + await vi.advanceTimersByTimeAsync(AUTO_RELOAD_INTERVAL); + expect(counts).toEqual({ storage: 3, scm: 3, om: 3, dn: 4 }); + } finally { + vi.useRealTimers(); + sessionStorage.removeItem('autoReloadEnabled'); + } + }); });
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/pages/capacity/capacity.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/pages/capacity/capacity.tsx index d6695af..f55e0e5 100644 --- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/pages/capacity/capacity.tsx +++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/pages/capacity/capacity.tsx
@@ -38,7 +38,6 @@ import * as CONSTANTS from '@/v2/constants/capacity.constants'; import { UtilizationResponse, SCMPendingDeletion, OMPendingDeletion, DNPendingDeletion, DataNodeUsage } from '@/v2/types/capacity.types'; import { useAutoReload } from '@/v2/hooks/useAutoReload.hook'; -import { AUTO_RELOAD_INTERVAL_DEFAULT } from '@/constants/autoReload.constants'; type CapacityState = { lastUpdated: number; @@ -101,11 +100,20 @@ return storageDistribution.data.dataNodeUsage.filter(dn => pendingHostNames.has(dn.hostName)); }, [storageDistribution.data.dataNodeUsage, dnPendingDeletes.data.pendingDeletionPerDataNode]); - // Seed selected datanode once data loads so dependent calculations work + // Seed selected datanode once data loads so dependent calculations work. + // Prefer the first available DN so the user does not land on the error card + // while a healthy DN exists; fall back to the first DN (surfacing the error + // card) only when every DN is unavailable. React.useEffect(() => { const hostNames = filteredDNs.map(dn => dn.hostName); if (!hostNames.includes(selectedDatanode)) { - setSelectedDatanode(hostNames[0] ?? ""); + const unavailableHosts = new Set( + (dnPendingDeletes.data.pendingDeletionPerDataNode ?? []) + .filter(dn => dn.pendingBlockSize === -1) + .map(dn => dn.hostName) + ); + const firstAvailable = hostNames.find(hostName => !unavailableHosts.has(hostName)); + setSelectedDatanode(firstAvailable ?? hostNames[0] ?? ""); } }, [filteredDNs]); @@ -133,7 +141,10 @@ const autoReload = useAutoReload(loadDataIfIdle); - const selectedDNDetails: DataNodeUsage & { pendingBlockSize: number } = React.useMemo(() => { + const selectedDNDetails: DataNodeUsage & { + pendingBlockSize: number; + unavailable: boolean; + } = React.useMemo(() => { const selected = storageDistribution.data.dataNodeUsage.find(datanode => datanode.hostName === selectedDatanode) ?? storageDistribution.data.dataNodeUsage[0]; const dnPendingEntry = dnPendingDeletes.data.pendingDeletionPerDataNode?.find( @@ -151,7 +162,10 @@ reserved: 0 }), ...dnPendingEntry, - pendingBlockSize: Math.max(0, dnPendingEntry.pendingBlockSize) + pendingBlockSize: Math.max(0, dnPendingEntry.pendingBlockSize), + // -1 is the sentinel for a DN whose pending deletion query failed (offline/unreachable). + // Its capacity data from the storage report may be outdated, so surface an error instead. + unavailable: dnPendingEntry.pendingBlockSize === -1 } }, [selectedDatanode, storageDistribution.data.dataNodeUsage, dnPendingDeletes.data.pendingDeletionPerDataNode]); @@ -202,19 +216,58 @@ } }; - // Adjust the polling interval based on DN scan status: - // fast (5s) while a scan is running, normal (60s) once finished. - // Honors the auto-reload toggle: if polling is OFF, do nothing. - React.useEffect(() => { - if (!autoReload.isPolling) { - return; + // --- DN pending-deletion scan polling -------------------------------------- + // The periodic full refresh (all four endpoints) is driven by useAutoReload + // when Auto Refresh is enabled, and by the manual reload button otherwise. + // On top of that, whenever a DN scan is reported IN_PROGRESS we poll ONLY the + // DN endpoint at a fast cadence until it finishes, then refetch the other + // endpoints once so the aggregate pending-deletion numbers reflect the final + // DN data. This runs regardless of the Auto Refresh toggle, because a running + // scan must be driven to completion either way. + + // Refetch only the DN endpoint. Kept in a ref so the interval below always + // reads the current `loading` value instead of a stale closure (the effect + // does not re-run while the status stays IN_PROGRESS). + const pollDnScanRef = React.useRef<() => void>(() => {}); + pollDnScanRef.current = () => { + if (!dnPendingDeletes.loading) { + dnPendingDeletes.refetch(); } - autoReload.startPolling( - dnPendingDeletes.data.status === "FINISHED" - ? AUTO_RELOAD_INTERVAL_DEFAULT - : PENDING_POLL_INTERVAL - ); - }, [dnPendingDeletes.data.status, autoReload.isPolling]); // eslint-disable-line react-hooks/exhaustive-deps + }; + + // Refetch everything except the DN endpoint, to re-sync the OM / SCM / storage + // numbers with the DN data once the scan has finished. + const syncNonDnDataRef = React.useRef<() => void>(() => {}); + syncNonDnDataRef.current = () => { + storageDistribution.refetch(); + scmPendingDeletes.refetch(); + omPendingDeletes.refetch(); + setState({ lastUpdated: Number(moment()) }); + }; + + // True while we are actively driving a DN scan, so that we sync the other + // endpoints exactly once when it transitions to FINISHED — and not on the + // initial load, where all four endpoints were already fetched together. + const isDrivingDnScanRef = React.useRef(false); + + React.useEffect(() => { + const scanInProgress = dnPendingDeletes.data.status === "IN_PROGRESS"; + + if (scanInProgress) { + isDrivingDnScanRef.current = true; + const timer = window.setInterval(() => pollDnScanRef.current(), PENDING_POLL_INTERVAL); + return () => clearInterval(timer); + } + + // Reached a terminal state (FINISHED / FAILED). If we were driving a scan, + // sync the other endpoints once (only on success) and stop tracking it. + if (isDrivingDnScanRef.current) { + isDrivingDnScanRef.current = false; + if (dnPendingDeletes.data.status === "FINISHED") { + syncNonDnDataRef.current(); + } + } + }, [dnPendingDeletes.data.status]); // eslint-disable-line react-hooks/exhaustive-deps const dnReportStatus = ( (dnPendingDeletes.data.totalNodeQueriesFailed ?? 0) > 0 @@ -453,6 +506,7 @@ downloadUrl={DN_CSV_DOWNLOAD_URL} onDownloadClick={() => downloadCsv(DN_CSV_DOWNLOAD_URL)} handleSelect={setSelectedDatanode} + selectedValue={selectedDatanode} dropdownItems={filteredDNs.map(datanode => ({ label: ( <> @@ -472,6 +526,9 @@ dataDetails={[{ title: 'USED SPACE', size: (selectedDNDetails.used ?? 0) + (selectedDNDetails.pendingBlockSize ?? 0), + hasError: selectedDNDetails.unavailable, + errorMessage: 'Datanode is unavailable; capacity data may be outdated.', + errorTestId: 'dn-used-space-error', breakdown: [{ label: 'PENDING DELETION', value: selectedDNDetails.pendingBlockSize ?? 0, @@ -484,6 +541,9 @@ }, { title: 'FREE SPACE', size: (selectedDNDetails.remaining ?? 0) + (selectedDNDetails.committed ?? 0), + hasError: selectedDNDetails.unavailable, + errorMessage: 'Datanode is unavailable; capacity data may be outdated.', + errorTestId: 'dn-free-space-error', breakdown: [{ label: unusedSpaceBreakdown, value: selectedDNDetails.remaining ?? 0,
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/pages/capacity/components/CapacityDetail.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/pages/capacity/components/CapacityDetail.tsx index f17c52b..3445c6e 100644 --- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/pages/capacity/components/CapacityDetail.tsx +++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/pages/capacity/components/CapacityDetail.tsx
@@ -50,6 +50,7 @@ disabledOpts?: string[]; optsClass?: string; handleSelect?: React.Dispatch<React.SetStateAction<string>> + selectedValue?: string; loading: boolean; extra?: React.ReactNode; }; @@ -111,6 +112,7 @@ optsClass, dataDetails, handleSelect, + selectedValue, loading, extra } @@ -149,7 +151,7 @@ {selectorTitle} <Select showSearch - defaultValue={options?.[0]?.value} + value={selectedValue ?? options?.[0]?.value} options={options} onChange={handleSelect} style={{ marginBottom: '16px' }}
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java index de8487d..81aea09 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java
@@ -268,6 +268,42 @@ public static void writeKeyToOm(OMMetadataManager omMetadataManager, .build()); } + /** + * Write a multi-block key on OM instance with an explicit replication config. + * Combines the location-group and replication-config overloads so tests can + * exercise a real (> 1) replication factor. + * @throw IOException while writing. + */ + @SuppressWarnings("checkstyle:parameternumber") + public static void writeKeyToOm(OMMetadataManager omMetadataManager, + String keyName, + String bucketName, + String volName, + String fileName, + long objectId, + long parentObjectId, + long bucketObjectId, + long volumeObjectId, + List<OmKeyLocationInfoGroup> locationVersions, + BucketLayout bucketLayout, + long dataSize, + ReplicationConfig replicationConfig) + throws IOException { + String omKey = getKey(omMetadataManager, keyName, bucketName, volName, + fileName, parentObjectId, bucketObjectId, volumeObjectId, bucketLayout); + omMetadataManager.getKeyTable(bucketLayout).put(omKey, + new OmKeyInfo.Builder() + .setBucketName(bucketName) + .setVolumeName(volName) + .setKeyName(keyName) + .setDataSize(dataSize) + .setOmKeyLocationInfos(locationVersions) + .setReplicationConfig(replicationConfig) + .setObjectID(objectId) + .setParentObjectID(parentObjectId) + .build()); + } + @SuppressWarnings("checkstyle:ParameterNumber") private static String getKey(OMMetadataManager omMetadataManager, String key, String bucket, String volume, String fileName, long parentObjectId, long bucketObjectId, long volumeObjectId,
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/AbstractTreeNSSummaryScenario.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/AbstractTreeNSSummaryScenario.java new file mode 100644 index 0000000..114647c --- /dev/null +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/AbstractTreeNSSummaryScenario.java
@@ -0,0 +1,806 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api; + +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; +import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; +import static org.apache.hadoop.ozone.om.helpers.QuotaUtil.getReplicatedSize; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; +import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType.USER; +import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.WRITE; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.StorageType; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.ozone.OzoneAcl; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; +import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.recon.api.types.BucketObjectDBInfo; +import org.apache.hadoop.ozone.recon.api.types.DUResponse; +import org.apache.hadoop.ozone.recon.api.types.EntityType; +import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse; +import org.apache.hadoop.ozone.recon.api.types.QuotaUsageResponse; +import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; +import org.apache.hadoop.ozone.recon.api.types.VolumeObjectDBInfo; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; + +/** + * Shared tree fixture for the FSO and Legacy (filesystem-paths enabled) + * scenarios. The namespace is: + * <pre> + * vol + * / \ + * bucket1 bucket2 + * / \ / \ + * file1 dir1 file4 file5 + * / \ \ \ + * dir2 dir3 dir4 file7 + * / \ \ + * file2 file3 file6 + * ---------------------------------------- + * vol2 + * / \ + * bucket3 bucket4 + * / \ / + * file8 dir5 file11 + * / \ + * file9 file10 + * ---------------------------------------- + * vol3 + * | + * bucket5 + * / \ + * file12 dir6 + * / \ + * file13 dir7 + * / + * file14 + * </pre> + * + * <p>Every key is written with RATIS/THREE replication so that + * {@code sizeWithReplica} (3x) is genuinely distinct from the unreplicated + * {@code size}. FSO and Legacy differ only in the OM DB key scheme, exposed + * through the {@code writeDir}/{@code writeKey}/{@code writeReplicatedKey} + * hooks; all constants and assertions are shared here. + */ +public abstract class AbstractTreeNSSummaryScenario extends NSSummaryTestScenario { + + // object names + static final String VOL = "vol"; + static final String VOL_TWO = "vol2"; + static final String VOL_THREE = "vol3"; + static final String BUCKET_ONE = "bucket1"; + static final String BUCKET_TWO = "bucket2"; + static final String BUCKET_THREE = "bucket3"; + static final String BUCKET_FOUR = "bucket4"; + static final String BUCKET_FIVE = "bucket5"; + static final String KEY_ONE = "file1"; + static final String KEY_TWO = "dir1/dir2/file2"; + static final String KEY_THREE = "dir1/dir3/file3"; + static final String KEY_FOUR = "file4"; + static final String KEY_FIVE = "file5"; + static final String KEY_SIX = "dir1/dir4/file6"; + static final String KEY_SEVEN = "dir1/file7"; + static final String KEY_EIGHT = "file8"; + static final String KEY_NINE = "dir5/file9"; + static final String KEY_TEN = "dir5/file10"; + static final String KEY_ELEVEN = "file11"; + static final String KEY_TWELVE = "file12"; + static final String KEY_THIRTEEN = "dir6/file13"; + static final String KEY_FOURTEEN = "dir6/dir7/file14"; + static final String FILE_ONE = "file1"; + static final String FILE_TWO = "file2"; + static final String FILE_THREE = "file3"; + static final String FILE_FOUR = "file4"; + static final String FILE_FIVE = "file5"; + static final String FILE_SIX = "file6"; + static final String FILE_SEVEN = "file7"; + static final String FILE_EIGHT = "file8"; + static final String FILE_NINE = "file9"; + static final String FILE_TEN = "file10"; + static final String FILE_ELEVEN = "file11"; + static final String FILE_TWELVE = "file12"; + static final String FILE_THIRTEEN = "file13"; + static final String FILE_FOURTEEN = "file14"; + static final String DIR_ONE = "dir1"; + static final String DIR_TWO = "dir2"; + static final String DIR_THREE = "dir3"; + static final String DIR_FOUR = "dir4"; + static final String DIR_FIVE = "dir5"; + static final String DIR_SIX = "dir6"; + static final String DIR_SEVEN = "dir7"; + + // object IDs + static final long VOL_OBJECT_ID = 0L; + static final long BUCKET_ONE_OBJECT_ID = 1L; + static final long BUCKET_TWO_OBJECT_ID = 2L; + static final long KEY_ONE_OBJECT_ID = 3L; + static final long DIR_ONE_OBJECT_ID = 4L; + static final long KEY_TWO_OBJECT_ID = 5L; + static final long KEY_FOUR_OBJECT_ID = 6L; + static final long DIR_TWO_OBJECT_ID = 7L; + static final long KEY_THREE_OBJECT_ID = 8L; + static final long KEY_FIVE_OBJECT_ID = 9L; + static final long KEY_SIX_OBJECT_ID = 10L; + static final long DIR_THREE_OBJECT_ID = 11L; + static final long DIR_FOUR_OBJECT_ID = 12L; + static final long KEY_SEVEN_OBJECT_ID = 13L; + static final long VOL_TWO_OBJECT_ID = 14L; + static final long BUCKET_THREE_OBJECT_ID = 15L; + static final long BUCKET_FOUR_OBJECT_ID = 16L; + static final long KEY_EIGHT_OBJECT_ID = 17L; + static final long DIR_FIVE_OBJECT_ID = 18L; + static final long KEY_NINE_OBJECT_ID = 19L; + static final long KEY_TEN_OBJECT_ID = 20L; + static final long KEY_ELEVEN_OBJECT_ID = 21L; + static final long VOL_THREE_OBJECT_ID = 22L; + static final long DIR_SIX_OBJECT_ID = 23L; + static final long DIR_SEVEN_OBJECT_ID = 24L; + static final long FILE_TWELVE_OBJECT_ID = 25L; + static final long FILE_THIRTEEN_OBJECT_ID = 26L; + static final long FILE_FOURTEEN_OBJECT_ID = 27L; + static final long BUCKET_FIVE_OBJECT_ID = 28L; + + // data size in bytes (unreplicated) + static final long KEY_ONE_SIZE = 500L; // bin 0 + static final long KEY_TWO_SIZE = OzoneConsts.KB + 1; // bin 1 + static final long KEY_THREE_SIZE = 4 * OzoneConsts.KB + 1; // bin 3 + static final long KEY_FOUR_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 + static final long KEY_FIVE_SIZE = 100L; // bin 0 + static final long KEY_SIX_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 + static final long KEY_SEVEN_SIZE = 4 * OzoneConsts.KB + 1; + static final long KEY_EIGHT_SIZE = OzoneConsts.KB + 1; // bin 1 + static final long KEY_NINE_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 + static final long KEY_TEN_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 + static final long KEY_ELEVEN_SIZE = OzoneConsts.KB + 1; // bin 1 + static final long KEY_TWELVE_SIZE = OzoneConsts.KB; + static final long KEY_THIRTEEN_SIZE = OzoneConsts.KB; + static final long KEY_FOURTEEN_SIZE = OzoneConsts.KB; + + // replicated sizes (RATIS/THREE => 3x the unreplicated size) + static final long FILE1_SIZE_WITH_REPLICA = repl(KEY_ONE_SIZE); + static final long FILE2_SIZE_WITH_REPLICA = repl(KEY_TWO_SIZE); + static final long FILE3_SIZE_WITH_REPLICA = repl(KEY_THREE_SIZE); + static final long FILE4_SIZE_WITH_REPLICA = repl(KEY_FOUR_SIZE); + static final long FILE5_SIZE_WITH_REPLICA = repl(KEY_FIVE_SIZE); + static final long FILE6_SIZE_WITH_REPLICA = repl(KEY_SIX_SIZE); + static final long FILE7_SIZE_WITH_REPLICA = repl(KEY_SEVEN_SIZE); + static final long FILE8_SIZE_WITH_REPLICA = repl(KEY_EIGHT_SIZE); + static final long FILE9_SIZE_WITH_REPLICA = repl(KEY_NINE_SIZE); + static final long FILE10_SIZE_WITH_REPLICA = repl(KEY_TEN_SIZE); + static final long FILE11_SIZE_WITH_REPLICA = repl(KEY_ELEVEN_SIZE); + static final long FILE12_SIZE_WITH_REPLICA = repl(KEY_TWELVE_SIZE); + static final long FILE13_SIZE_WITH_REPLICA = repl(KEY_THIRTEEN_SIZE); + static final long FILE14_SIZE_WITH_REPLICA = repl(KEY_FOURTEEN_SIZE); + + static final long REPLICA_UNDER_ROOT = + FILE1_SIZE_WITH_REPLICA + FILE2_SIZE_WITH_REPLICA + FILE3_SIZE_WITH_REPLICA + + FILE4_SIZE_WITH_REPLICA + FILE5_SIZE_WITH_REPLICA + FILE6_SIZE_WITH_REPLICA + + FILE7_SIZE_WITH_REPLICA + FILE8_SIZE_WITH_REPLICA + FILE9_SIZE_WITH_REPLICA + + FILE10_SIZE_WITH_REPLICA + FILE11_SIZE_WITH_REPLICA + FILE12_SIZE_WITH_REPLICA + + FILE13_SIZE_WITH_REPLICA + FILE14_SIZE_WITH_REPLICA; + static final long REPLICA_UNDER_VOL = + FILE1_SIZE_WITH_REPLICA + FILE2_SIZE_WITH_REPLICA + FILE3_SIZE_WITH_REPLICA + + FILE4_SIZE_WITH_REPLICA + FILE5_SIZE_WITH_REPLICA + FILE6_SIZE_WITH_REPLICA + + FILE7_SIZE_WITH_REPLICA; + static final long REPLICA_UNDER_VOL_TWO = + FILE8_SIZE_WITH_REPLICA + FILE9_SIZE_WITH_REPLICA + FILE10_SIZE_WITH_REPLICA + + FILE11_SIZE_WITH_REPLICA; + static final long REPLICA_UNDER_VOL_THREE = + FILE12_SIZE_WITH_REPLICA + FILE13_SIZE_WITH_REPLICA + FILE14_SIZE_WITH_REPLICA; + static final long REPLICA_UNDER_BUCKET1 = + FILE1_SIZE_WITH_REPLICA + FILE2_SIZE_WITH_REPLICA + FILE3_SIZE_WITH_REPLICA + + FILE6_SIZE_WITH_REPLICA + FILE7_SIZE_WITH_REPLICA; + static final long REPLICA_UNDER_BUCKET2 = + FILE4_SIZE_WITH_REPLICA + FILE5_SIZE_WITH_REPLICA; + static final long REPLICA_UNDER_DIR1 = + FILE2_SIZE_WITH_REPLICA + FILE3_SIZE_WITH_REPLICA + FILE6_SIZE_WITH_REPLICA + + FILE7_SIZE_WITH_REPLICA; + static final long REPLICA_UNDER_DIR2 = FILE2_SIZE_WITH_REPLICA; + static final long REPLICA_UNDER_KEY = FILE4_SIZE_WITH_REPLICA; + static final long REPLICA_UNDER_DIR6 = + FILE13_SIZE_WITH_REPLICA + FILE14_SIZE_WITH_REPLICA; + + // quota in bytes + static final long ROOT_QUOTA = 2 * (2 * OzoneConsts.MB); + static final long VOL_QUOTA = 2 * OzoneConsts.MB; + static final long VOL_TWO_QUOTA = 2 * OzoneConsts.MB; + static final long VOL_THREE_QUOTA = 2 * OzoneConsts.MB; + static final long BUCKET_ONE_QUOTA = OzoneConsts.MB; + static final long BUCKET_TWO_QUOTA = OzoneConsts.MB; + static final long BUCKET_THREE_QUOTA = OzoneConsts.MB; + static final long BUCKET_FOUR_QUOTA = OzoneConsts.MB; + static final long BUCKET_FIVE_QUOTA = OzoneConsts.MB; + + // request paths + static final String VOL_PATH = "/vol"; + static final String VOL_TWO_PATH = "/vol2"; + static final String VOL_THREE_PATH = "/vol3"; + static final String BUCKET_ONE_PATH = "/vol/bucket1"; + static final String BUCKET_TWO_PATH = "/vol/bucket2"; + static final String DIR_ONE_PATH = "/vol/bucket1/dir1"; + static final String DIR_TWO_PATH = "/vol/bucket1/dir1/dir2"; + static final String DIR_THREE_PATH = "/vol/bucket1/dir1/dir3"; + static final String DIR_FOUR_PATH = "/vol/bucket1/dir1/dir4"; + static final String MULTI_BLOCK_KEY_PATH = "/vol/bucket1/dir1/file7"; + + // unreplicated (namespace) data sizes used by non-replica DU and quota + static final long ROOT_DATA_SIZE = REPLICA_UNDER_ROOT; + static final long VOL_DATA_SIZE = KEY_ONE_SIZE + KEY_TWO_SIZE + KEY_THREE_SIZE + + KEY_FOUR_SIZE + KEY_FIVE_SIZE + KEY_SIX_SIZE + KEY_SEVEN_SIZE; + static final long VOL_TWO_DATA_SIZE = + KEY_EIGHT_SIZE + KEY_NINE_SIZE + KEY_TEN_SIZE + KEY_ELEVEN_SIZE; + static final long BUCKET_ONE_DATA_SIZE = KEY_ONE_SIZE + KEY_TWO_SIZE + + KEY_THREE_SIZE + KEY_SIX_SIZE + KEY_SEVEN_SIZE; + static final long BUCKET_TWO_DATA_SIZE = KEY_FOUR_SIZE + KEY_FIVE_SIZE; + static final long DIR_ONE_DATA_SIZE = + KEY_TWO_SIZE + KEY_THREE_SIZE + KEY_SIX_SIZE + KEY_SEVEN_SIZE; + + protected AbstractTreeNSSummaryScenario(String displayName) { + super(displayName); + } + + private static long repl(long size) { + return getReplicatedSize(size, RatisReplicationConfig.getInstance(THREE)); + } + + abstract BucketLayout getBucketLayout(); + + /** FSO leaves this alone; Legacy enables filesystem paths. */ + abstract void configureOmConfiguration(OzoneConfiguration conf); + + abstract void reprocess(ReconNamespaceSummaryManager reconNamespaceSummaryManager, + ReconOMMetadataManager reconOMMetadataManager, OzoneConfiguration conf) + throws IOException; + + /** + * Write a directory. {@code legacyPath} and {@code fsoParentObjectId} let the + * FSO (directory table, real parent) and Legacy (key table, path-based) + * schemes coexist behind one call site. + */ + @SuppressWarnings("checkstyle:ParameterNumber") + abstract void writeDir(ReconOMMetadataManager recon, String dirName, + String legacyPath, long dirObjectId, long fsoParentObjectId, + String bucketName, String volName, long bucketObjectId, long volObjectId) + throws IOException; + + @SuppressWarnings("checkstyle:ParameterNumber") + abstract void writeReplicatedKey(ReconOMMetadataManager recon, String key, + String bucket, String vol, String fileName, long keyObjectId, + long fsoParentObjectId, long bucketObjectId, long volObjectId, + OmKeyLocationInfoGroup locationGroup, long dataSize) throws IOException; + + @Override + boolean hasDirectories() { + return true; + } + + @Override + boolean hasVolumeThree() { + return true; + } + + @Override + long rootQuota() { + return ROOT_QUOTA; + } + + @Override + long rootDataSize() { + return ROOT_DATA_SIZE; + } + + @Override + OMMetadataManager initializeOmMetadataManager(File omDbDir, + OzoneConfiguration conf) throws IOException { + conf.set(OZONE_OM_DB_DIRS, omDbDir.getAbsolutePath()); + configureOmConfiguration(conf); + OMMetadataManager omMetadataManager = new OmMetadataManagerImpl(conf, null); + + putVolume(omMetadataManager, VOL, VOL_OBJECT_ID, VOL_QUOTA); + putVolume(omMetadataManager, VOL_TWO, VOL_TWO_OBJECT_ID, VOL_TWO_QUOTA); + putVolume(omMetadataManager, VOL_THREE, VOL_THREE_OBJECT_ID, VOL_THREE_QUOTA); + + putBucket(omMetadataManager, VOL, BUCKET_ONE, BUCKET_ONE_OBJECT_ID, + BUCKET_ONE_QUOTA); + putBucket(omMetadataManager, VOL, BUCKET_TWO, BUCKET_TWO_OBJECT_ID, + BUCKET_TWO_QUOTA); + putBucket(omMetadataManager, VOL_TWO, BUCKET_THREE, BUCKET_THREE_OBJECT_ID, + BUCKET_THREE_QUOTA); + putBucket(omMetadataManager, VOL_TWO, BUCKET_FOUR, BUCKET_FOUR_OBJECT_ID, + BUCKET_FOUR_QUOTA); + putBucket(omMetadataManager, VOL_THREE, BUCKET_FIVE, BUCKET_FIVE_OBJECT_ID, + BUCKET_FIVE_QUOTA); + return omMetadataManager; + } + + private void putVolume(OMMetadataManager omMetadataManager, String volume, + long objectId, long quota) throws IOException { + omMetadataManager.getVolumeTable().put( + omMetadataManager.getVolumeKey(volume), + OmVolumeArgs.newBuilder() + .setObjectID(objectId) + .setVolume(volume) + .setAdminName(TEST_USER) + .setOwnerName(TEST_USER) + .setQuotaInBytes(quota) + .build()); + } + + private void putBucket(OMMetadataManager omMetadataManager, String volume, + String bucket, long objectId, long quota) throws IOException { + OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() + .setVolumeName(volume) + .setBucketName(bucket) + .setObjectID(objectId) + .setQuotaInBytes(quota) + .setBucketLayout(getBucketLayout()) + .build(); + omMetadataManager.getBucketTable().put( + omMetadataManager.getBucketKey(volume, bucket), bucketInfo); + } + + @Override + void populateAndReprocess( + ReconNamespaceSummaryManager reconNamespaceSummaryManager, + ReconOMMetadataManager recon, OzoneConfiguration conf) throws Exception { + writeDirectories(recon); + writeVolumeThree(recon); + writeReplicatedKeys(recon); + reprocess(reconNamespaceSummaryManager, recon, conf); + } + + private void writeDirectories(ReconOMMetadataManager recon) + throws IOException { + String p = OzoneConsts.OM_KEY_PREFIX; + writeDir(recon, DIR_ONE, DIR_ONE + p, DIR_ONE_OBJECT_ID, + BUCKET_ONE_OBJECT_ID, BUCKET_ONE, VOL, BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID); + writeDir(recon, DIR_TWO, DIR_ONE + p + DIR_TWO + p, DIR_TWO_OBJECT_ID, + DIR_ONE_OBJECT_ID, BUCKET_ONE, VOL, BUCKET_ONE_OBJECT_ID, VOL_OBJECT_ID); + writeDir(recon, DIR_THREE, DIR_ONE + p + DIR_THREE + p, DIR_THREE_OBJECT_ID, + DIR_ONE_OBJECT_ID, BUCKET_ONE, VOL, BUCKET_ONE_OBJECT_ID, VOL_OBJECT_ID); + writeDir(recon, DIR_FOUR, DIR_ONE + p + DIR_FOUR + p, DIR_FOUR_OBJECT_ID, + DIR_ONE_OBJECT_ID, BUCKET_ONE, VOL, BUCKET_ONE_OBJECT_ID, VOL_OBJECT_ID); + writeDir(recon, DIR_FIVE, DIR_FIVE + p, DIR_FIVE_OBJECT_ID, + BUCKET_THREE_OBJECT_ID, BUCKET_THREE, VOL_TWO, BUCKET_THREE_OBJECT_ID, + VOL_TWO_OBJECT_ID); + } + + private void writeVolumeThree(ReconOMMetadataManager recon) + throws IOException { + String p = OzoneConsts.OM_KEY_PREFIX; + writeDir(recon, DIR_SIX, DIR_SIX + p, DIR_SIX_OBJECT_ID, + BUCKET_FIVE_OBJECT_ID, BUCKET_FIVE, VOL_THREE, BUCKET_FIVE_OBJECT_ID, + VOL_THREE_OBJECT_ID); + writeDir(recon, DIR_SEVEN, DIR_SIX + p + DIR_SEVEN + p, DIR_SEVEN_OBJECT_ID, + DIR_SIX_OBJECT_ID, BUCKET_FIVE, VOL_THREE, BUCKET_FIVE_OBJECT_ID, + VOL_THREE_OBJECT_ID); + // vol3 keys use the same (object-id parented) call for FSO and Legacy. + writeRatisKey(recon, KEY_TWELVE, BUCKET_FIVE, VOL_THREE, FILE_TWELVE, + FILE_TWELVE_OBJECT_ID, BUCKET_FIVE_OBJECT_ID, BUCKET_FIVE_OBJECT_ID, + VOL_THREE_OBJECT_ID, KEY_TWELVE_SIZE); + writeRatisKey(recon, KEY_THIRTEEN, BUCKET_FIVE, VOL_THREE, FILE_THIRTEEN, + FILE_THIRTEEN_OBJECT_ID, DIR_SIX_OBJECT_ID, BUCKET_FIVE_OBJECT_ID, + VOL_THREE_OBJECT_ID, KEY_THIRTEEN_SIZE); + writeRatisKey(recon, KEY_FOURTEEN, BUCKET_FIVE, VOL_THREE, FILE_FOURTEEN, + FILE_FOURTEEN_OBJECT_ID, DIR_SEVEN_OBJECT_ID, BUCKET_FIVE_OBJECT_ID, + VOL_THREE_OBJECT_ID, KEY_FOURTEEN_SIZE); + } + + @SuppressWarnings("checkstyle:ParameterNumber") + private void writeRatisKey(ReconOMMetadataManager recon, String key, + String bucket, String vol, String fileName, long objectId, + long parentObjectId, long bucketObjectId, long volObjectId, long dataSize) + throws IOException { + writeKeyToOm(recon, key, bucket, vol, fileName, objectId, parentObjectId, + bucketObjectId, volObjectId, dataSize, getBucketLayout(), + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), + org.apache.hadoop.util.Time.now(), true); + } + + private void writeReplicatedKeys(ReconOMMetadataManager recon) + throws IOException { + OmKeyLocationInfoGroup group1 = + NSSummaryEndpointTestBase.getLocationInfoGroup1(); + OmKeyLocationInfoGroup group2 = + NSSummaryEndpointTestBase.getLocationInfoGroup2(); + writeReplicatedKey(recon, KEY_ONE, BUCKET_ONE, VOL, FILE_ONE, + KEY_ONE_OBJECT_ID, BUCKET_ONE_OBJECT_ID, BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, group1, KEY_ONE_SIZE); + writeReplicatedKey(recon, KEY_TWO, BUCKET_ONE, VOL, FILE_TWO, + KEY_TWO_OBJECT_ID, DIR_TWO_OBJECT_ID, BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, group2, KEY_TWO_SIZE); + writeReplicatedKey(recon, KEY_THREE, BUCKET_ONE, VOL, FILE_THREE, + KEY_THREE_OBJECT_ID, DIR_THREE_OBJECT_ID, BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, group1, KEY_THREE_SIZE); + writeReplicatedKey(recon, KEY_FOUR, BUCKET_TWO, VOL, FILE_FOUR, + KEY_FOUR_OBJECT_ID, BUCKET_TWO_OBJECT_ID, BUCKET_TWO_OBJECT_ID, + VOL_OBJECT_ID, group2, KEY_FOUR_SIZE); + writeReplicatedKey(recon, KEY_FIVE, BUCKET_TWO, VOL, FILE_FIVE, + KEY_FIVE_OBJECT_ID, BUCKET_TWO_OBJECT_ID, BUCKET_TWO_OBJECT_ID, + VOL_OBJECT_ID, group1, KEY_FIVE_SIZE); + writeReplicatedKey(recon, KEY_SIX, BUCKET_ONE, VOL, FILE_SIX, + KEY_SIX_OBJECT_ID, DIR_FOUR_OBJECT_ID, BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, group2, KEY_SIX_SIZE); + writeReplicatedKey(recon, KEY_SEVEN, BUCKET_ONE, VOL, FILE_SEVEN, + KEY_SEVEN_OBJECT_ID, DIR_ONE_OBJECT_ID, BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, group1, KEY_SEVEN_SIZE); + writeReplicatedKey(recon, KEY_EIGHT, BUCKET_THREE, VOL_TWO, FILE_EIGHT, + KEY_EIGHT_OBJECT_ID, BUCKET_THREE_OBJECT_ID, BUCKET_THREE_OBJECT_ID, + VOL_TWO_OBJECT_ID, group2, KEY_EIGHT_SIZE); + writeReplicatedKey(recon, KEY_NINE, BUCKET_THREE, VOL_TWO, FILE_NINE, + KEY_NINE_OBJECT_ID, DIR_FIVE_OBJECT_ID, BUCKET_THREE_OBJECT_ID, + VOL_TWO_OBJECT_ID, group1, KEY_NINE_SIZE); + writeReplicatedKey(recon, KEY_TEN, BUCKET_THREE, VOL_TWO, FILE_TEN, + KEY_TEN_OBJECT_ID, DIR_FIVE_OBJECT_ID, BUCKET_THREE_OBJECT_ID, + VOL_TWO_OBJECT_ID, group2, KEY_TEN_SIZE); + writeReplicatedKey(recon, KEY_ELEVEN, BUCKET_FOUR, VOL_TWO, FILE_ELEVEN, + KEY_ELEVEN_OBJECT_ID, BUCKET_FOUR_OBJECT_ID, BUCKET_FOUR_OBJECT_ID, + VOL_TWO_OBJECT_ID, group1, KEY_ELEVEN_SIZE); + } + + @Override + void writeMultiBlockKey(ReconOMMetadataManager recon) throws IOException { + writeReplicatedKey(recon, KEY_SEVEN, BUCKET_ONE, VOL, FILE_SEVEN, + KEY_SEVEN_OBJECT_ID, DIR_ONE_OBJECT_ID, BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, NSSummaryEndpointTestBase.getLocationInfoGroup1(), + KEY_SEVEN_SIZE); + } + + @Override + String multiBlockKeyPath() { + return MULTI_BLOCK_KEY_PATH; + } + + @Override + long multiBlockKeyReplicatedSize() { + return FILE7_SIZE_WITH_REPLICA; + } + + // --------------------------------------------------------------------------- + // Assertions + // --------------------------------------------------------------------------- + + @Override + void assertBasicInfoRoot(NSSummaryEndpoint endpoint, + ReconOMMetadataManager recon) throws Exception { + String username = "myuser"; + OmPrefixInfo omPrefixInfo = OmPrefixInfo.newBuilder() + .setName(ROOT_PATH) + .setObjectID(10) + .setUpdateID(100) + .setAcls(singletonList(OzoneAcl.of(USER, username, ACCESS, WRITE))) + .addAllMetadata(singletonMap("key", "value")) + .build(); + recon.getPrefixTable().put(OzoneConsts.OM_KEY_PREFIX, omPrefixInfo); + + NamespaceSummaryResponse root = (NamespaceSummaryResponse) + endpoint.getBasicInfo(ROOT_PATH).getEntity(); + assertEquals(EntityType.ROOT, root.getEntityType()); + assertEquals(3, root.getCountStats().getNumVolume()); + assertEquals(5, root.getCountStats().getNumBucket()); + assertEquals(7, root.getCountStats().getNumTotalDir()); + assertEquals(14, root.getCountStats().getNumTotalKey()); + assertEquals("USER", root.getObjectDBInfo().getAcls().get(0).getType()); + assertEquals("WRITE", + root.getObjectDBInfo().getAcls().get(0).getAclList().get(0)); + assertEquals(username, root.getObjectDBInfo().getAcls().get(0).getName()); + assertEquals("value", root.getObjectDBInfo().getMetadata().get("key")); + assertEquals("ACCESS", root.getObjectDBInfo().getAcls().get(0).getScope()); + } + + @Override + void assertBasicInfoVolume(NSSummaryEndpoint endpoint) throws Exception { + NamespaceSummaryResponse vol = (NamespaceSummaryResponse) + endpoint.getBasicInfo(VOL_PATH).getEntity(); + assertEquals(EntityType.VOLUME, vol.getEntityType()); + assertEquals(2, vol.getCountStats().getNumBucket()); + assertEquals(4, vol.getCountStats().getNumTotalDir()); + assertEquals(7, vol.getCountStats().getNumTotalKey()); + assertEquals(TEST_USER, + ((VolumeObjectDBInfo) vol.getObjectDBInfo()).getAdmin()); + assertEquals(TEST_USER, + ((VolumeObjectDBInfo) vol.getObjectDBInfo()).getOwner()); + assertEquals(VOL, vol.getObjectDBInfo().getName()); + assertEquals(2097152, vol.getObjectDBInfo().getQuotaInBytes()); + assertEquals(-1, vol.getObjectDBInfo().getQuotaInNamespace()); + } + + @Override + void assertBasicInfoBucketOne(NSSummaryEndpoint endpoint) throws Exception { + NamespaceSummaryResponse bucket = (NamespaceSummaryResponse) + endpoint.getBasicInfo(BUCKET_ONE_PATH).getEntity(); + assertEquals(EntityType.BUCKET, bucket.getEntityType()); + assertEquals(4, bucket.getCountStats().getNumTotalDir()); + assertEquals(5, bucket.getCountStats().getNumTotalKey()); + assertEquals(VOL, + ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getVolumeName()); + assertEquals(StorageType.DISK, + ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getStorageType()); + assertEquals(getBucketLayout(), + ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getBucketLayout()); + assertEquals(BUCKET_ONE, + ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getName()); + } + + @Override + void assertBasicInfoBucketTwo(NSSummaryEndpoint endpoint) throws Exception { + NamespaceSummaryResponse bucket = (NamespaceSummaryResponse) + endpoint.getBasicInfo(BUCKET_TWO_PATH).getEntity(); + assertEquals(EntityType.BUCKET, bucket.getEntityType()); + assertEquals(0, bucket.getCountStats().getNumTotalDir()); + assertEquals(2, bucket.getCountStats().getNumTotalKey()); + assertEquals(VOL, + ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getVolumeName()); + assertEquals(StorageType.DISK, + ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getStorageType()); + assertEquals(getBucketLayout(), + ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getBucketLayout()); + assertEquals(BUCKET_TWO, + ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getName()); + } + + /** Directory basic info: identical for FSO and Legacy. */ + void assertBasicInfoDir(NSSummaryEndpoint endpoint) throws Exception { + NamespaceSummaryResponse dir = (NamespaceSummaryResponse) + endpoint.getBasicInfo(DIR_ONE_PATH).getEntity(); + assertEquals(EntityType.DIRECTORY, dir.getEntityType()); + assertEquals(3, dir.getCountStats().getNumTotalDir()); + assertEquals(4, dir.getCountStats().getNumTotalKey()); + assertEquals(DIR_ONE, dir.getObjectDBInfo().getName()); + assertEquals(0, dir.getObjectDBInfo().getMetadata().size()); + assertEquals(0, dir.getObjectDBInfo().getQuotaInBytes()); + assertEquals(0, dir.getObjectDBInfo().getQuotaInNamespace()); + assertEquals(0, dir.getObjectDBInfo().getUsedNamespace()); + } + + @Override + void assertDiskUsageRoot(NSSummaryEndpoint endpoint) throws Exception { + DUResponse duRootRes = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, ROOT_PATH, false); + assertEquals(3, duRootRes.getCount()); + List<DUResponse.DiskUsage> duData = duRootRes.getDuData(); + Collections.sort(duData, + Comparator.comparing(DUResponse.DiskUsage::getSubpath)); + assertEquals(VOL_PATH, duData.get(0).getSubpath()); + assertEquals(VOL_TWO_PATH, duData.get(1).getSubpath()); + assertEquals(VOL_DATA_SIZE, duData.get(0).getSize()); + assertEquals(VOL_TWO_DATA_SIZE, duData.get(1).getSize()); + } + + @Override + void assertDiskUsageVolume(NSSummaryEndpoint endpoint) throws Exception { + DUResponse duVolRes = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, VOL_PATH, false); + assertEquals(2, duVolRes.getCount()); + List<DUResponse.DiskUsage> duData = duVolRes.getDuData(); + Collections.sort(duData, + Comparator.comparing(DUResponse.DiskUsage::getSubpath)); + assertEquals(BUCKET_ONE_PATH, duData.get(0).getSubpath()); + assertEquals(BUCKET_TWO_PATH, duData.get(1).getSubpath()); + assertEquals(BUCKET_ONE_DATA_SIZE, duData.get(0).getSize()); + assertEquals(BUCKET_TWO_DATA_SIZE, duData.get(1).getSize()); + } + + /** Bucket-level DU (tree-only): bucket1 -> dir1. */ + void assertDiskUsageBucket(NSSummaryEndpoint endpoint) throws Exception { + DUResponse duBucketRes = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, BUCKET_ONE_PATH, false); + assertEquals(1, duBucketRes.getCount()); + DUResponse.DiskUsage duDir1 = duBucketRes.getDuData().get(0); + assertEquals(DIR_ONE_PATH, duDir1.getSubpath()); + assertEquals(DIR_ONE_DATA_SIZE, duDir1.getSize()); + } + + /** Directory-level DU (tree-only): dir1 -> dir2, dir3, dir4. */ + void assertDiskUsageDir(NSSummaryEndpoint endpoint) throws Exception { + DUResponse duDirRes = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, DIR_ONE_PATH, false); + assertEquals(3, duDirRes.getCount()); + List<DUResponse.DiskUsage> duSubDir = duDirRes.getDuData(); + Collections.sort(duSubDir, + Comparator.comparing(DUResponse.DiskUsage::getSubpath)); + assertEquals(DIR_TWO_PATH, duSubDir.get(0).getSubpath()); + assertEquals(KEY_TWO_SIZE, duSubDir.get(0).getSize()); + assertEquals(DIR_THREE_PATH, duSubDir.get(1).getSubpath()); + assertEquals(KEY_THREE_SIZE, duSubDir.get(1).getSize()); + assertEquals(DIR_FOUR_PATH, duSubDir.get(2).getSubpath()); + assertEquals(KEY_SIX_SIZE, duSubDir.get(2).getSize()); + } + + @Override + void assertDiskUsageKey(NSSummaryEndpoint endpoint) throws Exception { + DUResponse keyObj = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, KEY_PATH, false); + assertEquals(0, keyObj.getCount()); + assertEquals(KEY_FOUR_SIZE, keyObj.getSize()); + } + + @Override + void assertQuotaUsage(NSSummaryEndpoint endpoint) throws Exception { + QuotaUsageResponse quRootRes = (QuotaUsageResponse) + endpoint.getQuotaUsage(ROOT_PATH).getEntity(); + assertEquals(ROOT_QUOTA, quRootRes.getQuota()); + assertEquals(ROOT_DATA_SIZE, quRootRes.getQuotaUsed()); + + QuotaUsageResponse quVolRes = (QuotaUsageResponse) + endpoint.getQuotaUsage(VOL_PATH).getEntity(); + assertEquals(VOL_QUOTA, quVolRes.getQuota()); + assertEquals(VOL_DATA_SIZE, quVolRes.getQuotaUsed()); + + QuotaUsageResponse quBucketRes = (QuotaUsageResponse) + endpoint.getQuotaUsage(BUCKET_ONE_PATH).getEntity(); + assertEquals(BUCKET_ONE_QUOTA, quBucketRes.getQuota()); + assertEquals(BUCKET_ONE_DATA_SIZE, quBucketRes.getQuotaUsed()); + + QuotaUsageResponse quBucketRes2 = (QuotaUsageResponse) + endpoint.getQuotaUsage(BUCKET_TWO_PATH).getEntity(); + assertEquals(BUCKET_TWO_QUOTA, quBucketRes2.getQuota()); + assertEquals(BUCKET_TWO_DATA_SIZE, quBucketRes2.getQuotaUsed()); + + QuotaUsageResponse naDir = (QuotaUsageResponse) + endpoint.getQuotaUsage(DIR_ONE_PATH).getEntity(); + assertEquals(ResponseStatus.TYPE_NOT_APPLICABLE, naDir.getResponseCode()); + + QuotaUsageResponse naKey = (QuotaUsageResponse) + endpoint.getQuotaUsage(KEY_PATH).getEntity(); + assertEquals(ResponseStatus.TYPE_NOT_APPLICABLE, naKey.getResponseCode()); + + QuotaUsageResponse invalid = (QuotaUsageResponse) + endpoint.getQuotaUsage(INVALID_PATH).getEntity(); + assertEquals(ResponseStatus.PATH_NOT_FOUND, invalid.getResponseCode()); + } + + @Override + void assertFileSizeDist(NSSummaryEndpoint endpoint) throws Exception { + NSSummaryEndpointTestBase.checkFileSizeDist(endpoint, ROOT_PATH, 5, 3, 4, 2); + NSSummaryEndpointTestBase.checkFileSizeDist(endpoint, VOL_PATH, 2, 1, 2, 2); + NSSummaryEndpointTestBase.checkFileSizeDist( + endpoint, BUCKET_ONE_PATH, 1, 1, 1, 2); + NSSummaryEndpointTestBase.checkFileSizeDist( + endpoint, DIR_ONE_PATH, 0, 1, 1, 2); + } + + @Override + void assertDataSizeUnderRootWithReplication(NSSummaryEndpoint endpoint) + throws IOException { + DUResponse res = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, ROOT_PATH, true); + assertEquals(ResponseStatus.OK, res.getStatus()); + assertEquals(REPLICA_UNDER_ROOT, res.getSizeWithReplica()); + assertEquals(REPLICA_UNDER_VOL, res.getDuData().get(0).getSizeWithReplica()); + } + + @Override + void assertDataSizeUnderVolWithReplication(NSSummaryEndpoint endpoint) + throws IOException { + DUResponse res = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, VOL_PATH, true); + assertEquals(ResponseStatus.OK, res.getStatus()); + assertEquals(REPLICA_UNDER_VOL, res.getSizeWithReplica()); + assertEquals(REPLICA_UNDER_BUCKET1, + res.getDuData().get(0).getSizeWithReplica()); + } + + /** Bucket-level replicated DU (tree-only). */ + void assertDataSizeUnderBucketWithReplication(NSSummaryEndpoint endpoint) + throws IOException { + DUResponse res = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, BUCKET_ONE_PATH, true); + assertEquals(ResponseStatus.OK, res.getStatus()); + assertEquals(REPLICA_UNDER_BUCKET1, res.getSizeWithReplica()); + assertEquals(REPLICA_UNDER_DIR1, + res.getDuData().get(0).getSizeWithReplica()); + } + + /** Directory-level replicated DU (tree-only): dir1 -> dir2. */ + void assertDataSizeUnderDirWithReplication(NSSummaryEndpoint endpoint) + throws IOException { + DUResponse res = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, DIR_ONE_PATH, true); + assertEquals(ResponseStatus.OK, res.getStatus()); + assertEquals(REPLICA_UNDER_DIR1, res.getSizeWithReplica()); + assertEquals(REPLICA_UNDER_DIR2, + res.getDuData().get(0).getSizeWithReplica()); + } + + @Override + void assertDataSizeUnderKeyWithReplication(NSSummaryEndpoint endpoint) + throws IOException { + DUResponse res = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, KEY_PATH, true); + assertEquals(ResponseStatus.OK, res.getStatus()); + assertEquals(REPLICA_UNDER_KEY, res.getSizeWithReplica()); + } + + /** + * Replicated size propagates from files up through the directory hierarchy + * (tree-only). With RATIS/THREE keys, each level's replicated size is 3x its + * unreplicated size. + */ + void assertReplicatedSizePropagation(NSSummaryEndpoint endpoint) + throws IOException { + assertEquals(FILE2_SIZE_WITH_REPLICA, replicaSize(endpoint, DIR_TWO_PATH + "/file2")); + assertEquals(FILE3_SIZE_WITH_REPLICA, replicaSize(endpoint, DIR_THREE_PATH + "/file3")); + assertEquals(FILE6_SIZE_WITH_REPLICA, replicaSize(endpoint, DIR_FOUR_PATH + "/file6")); + assertEquals(FILE7_SIZE_WITH_REPLICA, replicaSize(endpoint, DIR_ONE_PATH + "/file7")); + + assertEquals(FILE2_SIZE_WITH_REPLICA, replicaSize(endpoint, DIR_TWO_PATH)); + assertEquals(FILE3_SIZE_WITH_REPLICA, replicaSize(endpoint, DIR_THREE_PATH)); + assertEquals(FILE6_SIZE_WITH_REPLICA, replicaSize(endpoint, DIR_FOUR_PATH)); + assertEquals(REPLICA_UNDER_DIR1, replicaSize(endpoint, DIR_ONE_PATH)); + assertEquals(REPLICA_UNDER_BUCKET1, replicaSize(endpoint, BUCKET_ONE_PATH)); + assertEquals(REPLICA_UNDER_BUCKET2, replicaSize(endpoint, BUCKET_TWO_PATH)); + assertEquals(REPLICA_UNDER_VOL, replicaSize(endpoint, VOL_PATH)); + assertEquals(REPLICA_UNDER_ROOT, replicaSize(endpoint, ROOT_PATH)); + } + + private long replicaSize(NSSummaryEndpoint endpoint, String path) + throws IOException { + return NSSummaryEndpointTestBase.getDiskUsage(endpoint, path, true) + .getSizeWithReplica(); + } + + /** vol3 RATIS DU (tree-only): asserts both the raw and 3x replicated size. */ + void assertRatisReplicationUnderVolumeThree(NSSummaryEndpoint endpoint) + throws IOException { + DUResponse res = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, VOL_THREE_PATH, true); + assertEquals(ResponseStatus.OK, res.getStatus()); + assertEquals(KEY_TWELVE_SIZE + KEY_THIRTEEN_SIZE + KEY_FOURTEEN_SIZE, + res.getSize()); + assertEquals(REPLICA_UNDER_VOL_THREE, res.getSizeWithReplica()); + } + + void assertRatisReplicationUnderBucketFive(NSSummaryEndpoint endpoint) + throws IOException { + DUResponse res = NSSummaryEndpointTestBase.getDiskUsage( + endpoint, VOL_THREE_PATH + "/" + BUCKET_FIVE, true); + assertEquals(ResponseStatus.OK, res.getStatus()); + assertEquals(KEY_TWELVE_SIZE + KEY_THIRTEEN_SIZE + KEY_FOURTEEN_SIZE, + res.getSize()); + assertEquals(REPLICA_UNDER_VOL_THREE, res.getSizeWithReplica()); + } + + void assertRatisReplicationUnderDirSix(NSSummaryEndpoint endpoint) + throws IOException { + DUResponse res = NSSummaryEndpointTestBase.getDiskUsage( + endpoint, VOL_THREE_PATH + "/" + BUCKET_FIVE + "/" + DIR_SIX, true); + assertEquals(ResponseStatus.OK, res.getStatus()); + assertEquals(KEY_THIRTEEN_SIZE + KEY_FOURTEEN_SIZE, res.getSize()); + assertEquals(REPLICA_UNDER_DIR6, res.getSizeWithReplica()); + } +}
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/FlatNSSummaryScenario.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/FlatNSSummaryScenario.java new file mode 100644 index 0000000..7bd37ab --- /dev/null +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/FlatNSSummaryScenario.java
@@ -0,0 +1,655 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api; + +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; +import static org.apache.hadoop.ozone.om.helpers.QuotaUtil.getReplicatedSize; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProvider; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.setConfiguration; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.StorageType; +import org.apache.hadoop.ozone.OmUtils; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.recon.ReconUtils; +import org.apache.hadoop.ozone.recon.api.types.BucketObjectDBInfo; +import org.apache.hadoop.ozone.recon.api.types.DUResponse; +import org.apache.hadoop.ozone.recon.api.types.EntityType; +import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse; +import org.apache.hadoop.ozone.recon.api.types.QuotaUsageResponse; +import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; +import org.apache.hadoop.ozone.recon.api.types.VolumeObjectDBInfo; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; +import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithLegacy; +import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithOBS; + +/** + * Flat fixture with OBS buckets (bucket1, bucket2 under vol) and Legacy buckets + * with filesystem paths disabled (bucket3, bucket4 under vol2), so both behave + * as flat object stores: + * <pre> + * └── vol + * ├── bucket1 (OBS) : file1, ////file2, file3/// + * └── bucket2 (OBS) : file4, _////// + * └── vol2 + * ├── bucket3 (Legacy): file8, //////, ///__file10 + * └── bucket4 (Legacy): ////file11 + * </pre> + * Every key is written with RATIS/THREE replication. + */ +public class FlatNSSummaryScenario extends NSSummaryTestScenario { + + // object names + private static final String VOL = "vol"; + private static final String VOL_TWO = "vol2"; + private static final String BUCKET_ONE = "bucket1"; + private static final String BUCKET_TWO = "bucket2"; + private static final String BUCKET_THREE = "bucket3"; + private static final String BUCKET_FOUR = "bucket4"; + private static final String KEY_ONE = "file1"; + private static final String KEY_TWO = "////file2"; + private static final String KEY_THREE = "file3///"; + private static final String KEY_FOUR = "file4"; + private static final String KEY_FIVE = "_//////"; + private static final String KEY_EIGHT = "file8"; + private static final String KEY_NINE = "//////"; + private static final String KEY_TEN = "///__file10"; + private static final String KEY_ELEVEN = "////file11"; + private static final String MULTI_BLOCK_FILE = KEY_THREE; + + // object IDs + private static final long VOL_OBJECT_ID = 0L; + private static final long VOL_TWO_OBJECT_ID = 14L; + private static final long BUCKET_ONE_OBJECT_ID = 1L; + private static final long BUCKET_TWO_OBJECT_ID = 2L; + private static final long BUCKET_THREE_OBJECT_ID = 15L; + private static final long BUCKET_FOUR_OBJECT_ID = 16L; + private static final long KEY_ONE_OBJECT_ID = 3L; + private static final long KEY_TWO_OBJECT_ID = 5L; + private static final long KEY_THREE_OBJECT_ID = 8L; + private static final long KEY_FOUR_OBJECT_ID = 6L; + private static final long KEY_FIVE_OBJECT_ID = 9L; + private static final long KEY_EIGHT_OBJECT_ID = 17L; + private static final long KEY_NINE_OBJECT_ID = 19L; + private static final long KEY_TEN_OBJECT_ID = 20L; + private static final long KEY_ELEVEN_OBJECT_ID = 21L; + private static final long MULTI_BLOCK_KEY_OBJECT_ID = 13L; + + // data size in bytes (unreplicated) + private static final long FILE_ONE_SIZE = 500L; // bin 0 + private static final long FILE_TWO_SIZE = OzoneConsts.KB + 1; // bin 1 + private static final long FILE_THREE_SIZE = 4 * OzoneConsts.KB + 1; // bin 3 + private static final long FILE_FOUR_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 + private static final long FILE_FIVE_SIZE = 100L; // bin 0 + private static final long FILE_EIGHT_SIZE = OzoneConsts.KB + 1; // bin 1 + private static final long FILE_NINE_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 + private static final long FILE_TEN_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 + private static final long FILE_ELEVEN_SIZE = OzoneConsts.KB + 1; // bin 1 + + // replicated sizes (RATIS/THREE => 3x) + private static final long FILE1_REPLICA = repl(FILE_ONE_SIZE); + private static final long FILE2_REPLICA = repl(FILE_TWO_SIZE); + private static final long FILE3_REPLICA = repl(FILE_THREE_SIZE); + private static final long FILE4_REPLICA = repl(FILE_FOUR_SIZE); + private static final long FILE5_REPLICA = repl(FILE_FIVE_SIZE); + private static final long FILE8_REPLICA = repl(FILE_EIGHT_SIZE); + private static final long FILE9_REPLICA = repl(FILE_NINE_SIZE); + private static final long FILE10_REPLICA = repl(FILE_TEN_SIZE); + private static final long FILE11_REPLICA = repl(FILE_ELEVEN_SIZE); + + private static final long REPLICA_UNDER_ROOT = + FILE1_REPLICA + FILE2_REPLICA + FILE3_REPLICA + FILE4_REPLICA + + FILE5_REPLICA + FILE8_REPLICA + FILE9_REPLICA + FILE10_REPLICA + + FILE11_REPLICA; + private static final long REPLICA_UNDER_VOL = + FILE1_REPLICA + FILE2_REPLICA + FILE3_REPLICA + FILE4_REPLICA + + FILE5_REPLICA; + private static final long REPLICA_UNDER_BUCKET1 = + FILE1_REPLICA + FILE2_REPLICA + FILE3_REPLICA; + private static final long REPLICA_UNDER_BUCKET3 = + FILE8_REPLICA + FILE9_REPLICA + FILE10_REPLICA; + private static final long REPLICA_UNDER_KEY = FILE4_REPLICA; + + // quota in bytes + private static final long ROOT_QUOTA = 2 * (2 * OzoneConsts.MB); + private static final long VOL_QUOTA = 2 * OzoneConsts.MB; + private static final long VOL_TWO_QUOTA = 2 * OzoneConsts.MB; + private static final long BUCKET_ONE_QUOTA = OzoneConsts.MB; + private static final long BUCKET_TWO_QUOTA = OzoneConsts.MB; + private static final long BUCKET_THREE_QUOTA = OzoneConsts.MB; + private static final long BUCKET_FOUR_QUOTA = OzoneConsts.MB; + + // request paths + private static final String VOL_PATH = "/vol"; + private static final String VOL_TWO_PATH = "/vol2"; + private static final String BUCKET_ONE_PATH = "/vol/bucket1"; + private static final String BUCKET_TWO_PATH = "/vol/bucket2"; + private static final String BUCKET_THREE_PATH = "/vol2/bucket3"; + private static final String BUCKET_FOUR_PATH = "/vol2/bucket4"; + private static final String KEY_ONE_PATH = "/vol/bucket1/" + KEY_ONE; + private static final String KEY_TWO_PATH = "/vol/bucket1/" + KEY_TWO; + private static final String KEY_FIVE_PATH = "/vol/bucket2/" + KEY_FIVE; + private static final String KEY_EIGHT_PATH = "/vol2/bucket3/" + KEY_EIGHT; + private static final String KEY_ELEVEN_PATH = "/vol2/bucket4/" + KEY_ELEVEN; + private static final String MULTI_BLOCK_KEY_PATH = "/vol/bucket1/" + KEY_THREE; + + // unreplicated (namespace) data sizes + private static final long ROOT_DATA_SIZE = REPLICA_UNDER_ROOT; + private static final long VOL_DATA_SIZE = FILE_ONE_SIZE + FILE_TWO_SIZE + + FILE_THREE_SIZE + FILE_FOUR_SIZE + FILE_FIVE_SIZE; + private static final long VOL_TWO_DATA_SIZE = + FILE_EIGHT_SIZE + FILE_NINE_SIZE + FILE_TEN_SIZE + FILE_ELEVEN_SIZE; + private static final long BUCKET_ONE_DATA_SIZE = + FILE_ONE_SIZE + FILE_TWO_SIZE + FILE_THREE_SIZE; + private static final long BUCKET_TWO_DATA_SIZE = + FILE_FOUR_SIZE + FILE_FIVE_SIZE; + private static final long BUCKET_THREE_DATA_SIZE = + FILE_EIGHT_SIZE + FILE_NINE_SIZE + FILE_TEN_SIZE; + private static final long BUCKET_FOUR_DATA_SIZE = FILE_ELEVEN_SIZE; + + public FlatNSSummaryScenario() { + super("OBS_AND_LEGACY"); + } + + private static long repl(long size) { + return getReplicatedSize(size, RatisReplicationConfig.getInstance(THREE)); + } + + private static BucketLayout obs() { + return BucketLayout.OBJECT_STORE; + } + + private static BucketLayout legacy() { + return BucketLayout.LEGACY; + } + + @Override + boolean isFlatLayout() { + return true; + } + + @Override + long rootQuota() { + return ROOT_QUOTA; + } + + @Override + long rootDataSize() { + return ROOT_DATA_SIZE; + } + + @Override + OzoneConfiguration newConfiguration() { + OzoneConfiguration conf = new OzoneConfiguration(); + // Legacy buckets behave like OBS buckets with filesystem paths disabled. + conf.set(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, "false"); + return conf; + } + + @Override + OzoneManagerServiceProviderImpl mockOmServiceProvider() throws IOException { + return getMockOzoneManagerServiceProvider(); + } + + @Override + OMMetadataManager initializeOmMetadataManager(File omDbDir, + OzoneConfiguration conf) throws IOException { + conf.set(OZONE_OM_DB_DIRS, omDbDir.getAbsolutePath()); + conf.set(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, "false"); + // Make the Recon metadata manager (and thus the endpoint's bucket-handler + // selection) observe filesystem-paths=false, so Legacy buckets are treated + // as flat object stores. Required because getTestReconOmMetadataManager + // reads a shared static configuration that a prior scenario may have set. + setConfiguration(conf); + OMMetadataManager omMetadataManager = new OmMetadataManagerImpl(conf, null); + + putVolume(omMetadataManager, VOL, VOL_OBJECT_ID, VOL_QUOTA); + putVolume(omMetadataManager, VOL_TWO, VOL_TWO_OBJECT_ID, VOL_TWO_QUOTA); + putBucket(omMetadataManager, VOL, BUCKET_ONE, BUCKET_ONE_OBJECT_ID, + BUCKET_ONE_QUOTA, obs()); + putBucket(omMetadataManager, VOL, BUCKET_TWO, BUCKET_TWO_OBJECT_ID, + BUCKET_TWO_QUOTA, obs()); + putBucket(omMetadataManager, VOL_TWO, BUCKET_THREE, BUCKET_THREE_OBJECT_ID, + BUCKET_THREE_QUOTA, legacy()); + putBucket(omMetadataManager, VOL_TWO, BUCKET_FOUR, BUCKET_FOUR_OBJECT_ID, + BUCKET_FOUR_QUOTA, legacy()); + return omMetadataManager; + } + + private void putVolume(OMMetadataManager omMetadataManager, String volume, + long objectId, long quota) throws IOException { + omMetadataManager.getVolumeTable().put( + omMetadataManager.getVolumeKey(volume), + OmVolumeArgs.newBuilder() + .setObjectID(objectId) + .setVolume(volume) + .setAdminName(TEST_USER) + .setOwnerName(TEST_USER) + .setQuotaInBytes(quota) + .build()); + } + + private void putBucket(OMMetadataManager omMetadataManager, String volume, + String bucket, long objectId, long quota, BucketLayout layout) + throws IOException { + omMetadataManager.getBucketTable().put( + omMetadataManager.getBucketKey(volume, bucket), + OmBucketInfo.newBuilder() + .setVolumeName(volume) + .setBucketName(bucket) + .setObjectID(objectId) + .setQuotaInBytes(quota) + .setBucketLayout(layout) + .build()); + } + + @Override + void populateAndReprocess( + ReconNamespaceSummaryManager reconNamespaceSummaryManager, + ReconOMMetadataManager recon, OzoneConfiguration conf) throws Exception { + OmKeyLocationInfoGroup group1 = + NSSummaryEndpointTestBase.getLocationInfoGroup1(); + OmKeyLocationInfoGroup group2 = + NSSummaryEndpointTestBase.getLocationInfoGroup2(); + + writeReplicatedKey(recon, KEY_ONE, BUCKET_ONE, VOL, KEY_ONE_OBJECT_ID, + BUCKET_ONE_OBJECT_ID, VOL_OBJECT_ID, group1, FILE_ONE_SIZE, obs()); + writeReplicatedKey(recon, KEY_TWO, BUCKET_ONE, VOL, KEY_TWO_OBJECT_ID, + BUCKET_ONE_OBJECT_ID, VOL_OBJECT_ID, group2, FILE_TWO_SIZE, obs()); + writeReplicatedKey(recon, KEY_THREE, BUCKET_ONE, VOL, KEY_THREE_OBJECT_ID, + BUCKET_ONE_OBJECT_ID, VOL_OBJECT_ID, group1, FILE_THREE_SIZE, obs()); + writeReplicatedKey(recon, KEY_FOUR, BUCKET_TWO, VOL, KEY_FOUR_OBJECT_ID, + BUCKET_TWO_OBJECT_ID, VOL_OBJECT_ID, group2, FILE_FOUR_SIZE, obs()); + writeReplicatedKey(recon, KEY_FIVE, BUCKET_TWO, VOL, KEY_FIVE_OBJECT_ID, + BUCKET_TWO_OBJECT_ID, VOL_OBJECT_ID, group1, FILE_FIVE_SIZE, obs()); + writeReplicatedKey(recon, KEY_EIGHT, BUCKET_THREE, VOL_TWO, + KEY_EIGHT_OBJECT_ID, BUCKET_THREE_OBJECT_ID, VOL_TWO_OBJECT_ID, group2, + FILE_EIGHT_SIZE, legacy()); + writeReplicatedKey(recon, KEY_NINE, BUCKET_THREE, VOL_TWO, + KEY_NINE_OBJECT_ID, BUCKET_THREE_OBJECT_ID, VOL_TWO_OBJECT_ID, group1, + FILE_NINE_SIZE, legacy()); + writeReplicatedKey(recon, KEY_TEN, BUCKET_THREE, VOL_TWO, KEY_TEN_OBJECT_ID, + BUCKET_THREE_OBJECT_ID, VOL_TWO_OBJECT_ID, group2, FILE_TEN_SIZE, + legacy()); + writeReplicatedKey(recon, KEY_ELEVEN, BUCKET_FOUR, VOL_TWO, + KEY_ELEVEN_OBJECT_ID, BUCKET_FOUR_OBJECT_ID, VOL_TWO_OBJECT_ID, group1, + FILE_ELEVEN_SIZE, legacy()); + + new NSSummaryTaskWithOBS(reconNamespaceSummaryManager, recon, 10, 5, 20, + 2000).reprocessWithOBS(recon); + new NSSummaryTaskWithLegacy(reconNamespaceSummaryManager, recon, conf, 10) + .reprocessWithLegacy(recon); + } + + @SuppressWarnings("checkstyle:ParameterNumber") + private void writeReplicatedKey(ReconOMMetadataManager recon, String key, + String bucket, String vol, long keyObjectId, long bucketObjectId, + long volObjectId, OmKeyLocationInfoGroup locationGroup, long dataSize, + BucketLayout layout) throws IOException { + writeKeyToOm(recon, key, bucket, vol, key, keyObjectId, + bucketObjectId, bucketObjectId, volObjectId, + Collections.singletonList(locationGroup), layout, dataSize, + RatisReplicationConfig.getInstance(THREE)); + } + + @Override + void writeMultiBlockKey(ReconOMMetadataManager recon) throws IOException { + writeReplicatedKey(recon, MULTI_BLOCK_FILE, BUCKET_ONE, VOL, + MULTI_BLOCK_KEY_OBJECT_ID, BUCKET_ONE_OBJECT_ID, VOL_OBJECT_ID, + NSSummaryEndpointTestBase.getLocationInfoGroup1(), FILE_THREE_SIZE, + obs()); + } + + @Override + String multiBlockKeyPath() { + return MULTI_BLOCK_KEY_PATH; + } + + @Override + long multiBlockKeyReplicatedSize() { + return FILE3_REPLICA; + } + + // --------------------------------------------------------------------------- + // Assertions common to every layout + // --------------------------------------------------------------------------- + + @Override + void assertBasicInfoRoot(NSSummaryEndpoint endpoint, + ReconOMMetadataManager recon) throws Exception { + NamespaceSummaryResponse root = (NamespaceSummaryResponse) + endpoint.getBasicInfo(ROOT_PATH).getEntity(); + assertEquals(EntityType.ROOT, root.getEntityType()); + assertEquals(2, root.getCountStats().getNumVolume()); + assertEquals(4, root.getCountStats().getNumBucket()); + assertEquals(9, root.getCountStats().getNumTotalKey()); + } + + @Override + void assertBasicInfoVolume(NSSummaryEndpoint endpoint) throws Exception { + NamespaceSummaryResponse vol = (NamespaceSummaryResponse) + endpoint.getBasicInfo(VOL_PATH).getEntity(); + assertEquals(EntityType.VOLUME, vol.getEntityType()); + assertEquals(2, vol.getCountStats().getNumBucket()); + assertEquals(5, vol.getCountStats().getNumTotalKey()); + assertEquals(TEST_USER, + ((VolumeObjectDBInfo) vol.getObjectDBInfo()).getAdmin()); + assertEquals(TEST_USER, + ((VolumeObjectDBInfo) vol.getObjectDBInfo()).getOwner()); + assertEquals(VOL, vol.getObjectDBInfo().getName()); + assertEquals(2097152, vol.getObjectDBInfo().getQuotaInBytes()); + assertEquals(-1, vol.getObjectDBInfo().getQuotaInNamespace()); + } + + @Override + void assertBasicInfoBucketOne(NSSummaryEndpoint endpoint) throws Exception { + assertObsBucket(endpoint, BUCKET_ONE_PATH, BUCKET_ONE, VOL, 3); + } + + @Override + void assertBasicInfoBucketTwo(NSSummaryEndpoint endpoint) throws Exception { + assertObsBucket(endpoint, BUCKET_TWO_PATH, BUCKET_TWO, VOL, 2); + } + + void assertBasicInfoVolTwo(NSSummaryEndpoint endpoint) throws Exception { + NamespaceSummaryResponse vol = (NamespaceSummaryResponse) + endpoint.getBasicInfo(VOL_TWO_PATH).getEntity(); + assertEquals(EntityType.VOLUME, vol.getEntityType()); + assertEquals(2, vol.getCountStats().getNumBucket()); + assertEquals(4, vol.getCountStats().getNumTotalKey()); + assertEquals(TEST_USER, + ((VolumeObjectDBInfo) vol.getObjectDBInfo()).getAdmin()); + assertEquals(TEST_USER, + ((VolumeObjectDBInfo) vol.getObjectDBInfo()).getOwner()); + assertEquals(VOL_TWO, vol.getObjectDBInfo().getName()); + assertEquals(2097152, vol.getObjectDBInfo().getQuotaInBytes()); + assertEquals(-1, vol.getObjectDBInfo().getQuotaInNamespace()); + } + + void assertBasicInfoBucketThree(NSSummaryEndpoint endpoint) throws Exception { + assertLegacyBucket(endpoint, BUCKET_THREE_PATH, BUCKET_THREE, VOL_TWO, 3); + } + + void assertBasicInfoBucketFour(NSSummaryEndpoint endpoint) throws Exception { + assertLegacyBucket(endpoint, BUCKET_FOUR_PATH, BUCKET_FOUR, VOL_TWO, 1); + } + + private void assertObsBucket(NSSummaryEndpoint endpoint, String path, + String bucketName, String volName, int numKeys) throws Exception { + assertBucket(endpoint, path, bucketName, volName, numKeys, obs()); + } + + private void assertLegacyBucket(NSSummaryEndpoint endpoint, String path, + String bucketName, String volName, int numKeys) throws Exception { + assertBucket(endpoint, path, bucketName, volName, numKeys, legacy()); + } + + private void assertBucket(NSSummaryEndpoint endpoint, String path, + String bucketName, String volName, int numKeys, BucketLayout layout) + throws Exception { + NamespaceSummaryResponse bucket = (NamespaceSummaryResponse) + endpoint.getBasicInfo(path).getEntity(); + assertEquals(EntityType.BUCKET, bucket.getEntityType()); + assertEquals(numKeys, bucket.getCountStats().getNumTotalKey()); + assertEquals(volName, + ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getVolumeName()); + assertEquals(StorageType.DISK, + ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getStorageType()); + assertEquals(layout, + ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getBucketLayout()); + assertEquals(bucketName, + ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getName()); + } + + @Override + void assertDiskUsageRoot(NSSummaryEndpoint endpoint) throws Exception { + DUResponse duRootRes = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, ROOT_PATH, false); + assertEquals(2, duRootRes.getCount()); + List<DUResponse.DiskUsage> duData = duRootRes.getDuData(); + Collections.sort(duData, + Comparator.comparing(DUResponse.DiskUsage::getSubpath)); + assertEquals(VOL_PATH, duData.get(0).getSubpath()); + assertEquals(VOL_TWO_PATH, duData.get(1).getSubpath()); + assertEquals(VOL_DATA_SIZE, duData.get(0).getSize()); + assertEquals(VOL_TWO_DATA_SIZE, duData.get(1).getSize()); + } + + @Override + void assertDiskUsageVolume(NSSummaryEndpoint endpoint) throws Exception { + DUResponse duVolRes = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, VOL_PATH, false); + assertEquals(2, duVolRes.getCount()); + List<DUResponse.DiskUsage> duData = duVolRes.getDuData(); + Collections.sort(duData, + Comparator.comparing(DUResponse.DiskUsage::getSubpath)); + assertEquals(BUCKET_ONE_PATH, duData.get(0).getSubpath()); + assertEquals(BUCKET_TWO_PATH, duData.get(1).getSubpath()); + assertEquals(BUCKET_ONE_DATA_SIZE, duData.get(0).getSize()); + assertEquals(BUCKET_TWO_DATA_SIZE, duData.get(1).getSize()); + } + + void assertDiskUsageVolTwo(NSSummaryEndpoint endpoint) throws Exception { + DUResponse duVolRes = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, VOL_TWO_PATH, false); + assertEquals(2, duVolRes.getCount()); + List<DUResponse.DiskUsage> duData = duVolRes.getDuData(); + Collections.sort(duData, + Comparator.comparing(DUResponse.DiskUsage::getSubpath)); + assertEquals(BUCKET_THREE_PATH, duData.get(0).getSubpath()); + assertEquals(BUCKET_FOUR_PATH, duData.get(1).getSubpath()); + assertEquals(VOL_TWO_DATA_SIZE, duVolRes.getSize()); + } + + /** OBS/Legacy buckets have no sub-paths unless files are explicitly listed. */ + void assertDiskUsageBucketOne(NSSummaryEndpoint endpoint) throws Exception { + assertFlatBucketDiskUsage(endpoint, BUCKET_ONE_PATH, 3, BUCKET_ONE_DATA_SIZE); + } + + void assertDiskUsageBucketTwo(NSSummaryEndpoint endpoint) throws Exception { + assertFlatBucketDiskUsage(endpoint, BUCKET_TWO_PATH, 2, BUCKET_TWO_DATA_SIZE); + } + + void assertDiskUsageBucketThree(NSSummaryEndpoint endpoint) throws Exception { + assertFlatBucketDiskUsage(endpoint, BUCKET_THREE_PATH, 3, + BUCKET_THREE_DATA_SIZE); + } + + private void assertFlatBucketDiskUsage(NSSummaryEndpoint endpoint, String path, + int numFiles, long dataSize) throws Exception { + DUResponse duBucketResponse = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, path, false); + assertEquals(0, duBucketResponse.getCount()); + DUResponse withFiles = (DUResponse) endpoint + .getDiskUsage(path, true, false, false).getEntity(); + assertEquals(numFiles, withFiles.getCount()); + assertEquals(dataSize, duBucketResponse.getSize()); + } + + @Override + void assertDiskUsageKey(NSSummaryEndpoint endpoint) throws Exception { + DUResponse duKeyResponse = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, KEY_PATH, false); + assertEquals(0, duKeyResponse.getCount()); + assertEquals(FILE_FOUR_SIZE, duKeyResponse.getSize()); + } + + void assertDiskUsageKeys(NSSummaryEndpoint endpoint) throws Exception { + assertKeySize(endpoint, KEY_ONE_PATH, FILE_ONE_SIZE); + assertKeySize(endpoint, KEY_TWO_PATH, FILE_TWO_SIZE); + assertKeySize(endpoint, KEY_FIVE_PATH, FILE_FIVE_SIZE); + assertKeySize(endpoint, KEY_EIGHT_PATH, FILE_EIGHT_SIZE); + assertKeySize(endpoint, KEY_ELEVEN_PATH, FILE_ELEVEN_SIZE); + } + + private void assertKeySize(NSSummaryEndpoint endpoint, String path, + long expectedSize) throws Exception { + DUResponse duKeyResponse = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, path, false); + assertEquals(0, duKeyResponse.getCount()); + assertEquals(expectedSize, duKeyResponse.getSize()); + } + + @Override + void assertQuotaUsage(NSSummaryEndpoint endpoint) throws Exception { + QuotaUsageResponse quRootRes = (QuotaUsageResponse) + endpoint.getQuotaUsage(ROOT_PATH).getEntity(); + assertEquals(ROOT_QUOTA, quRootRes.getQuota()); + assertEquals(ROOT_DATA_SIZE, quRootRes.getQuotaUsed()); + + assertQuota(endpoint, VOL_PATH, VOL_QUOTA, VOL_DATA_SIZE); + assertQuota(endpoint, BUCKET_ONE_PATH, BUCKET_ONE_QUOTA, + BUCKET_ONE_DATA_SIZE); + assertQuota(endpoint, BUCKET_TWO_PATH, BUCKET_TWO_QUOTA, + BUCKET_TWO_DATA_SIZE); + assertQuota(endpoint, BUCKET_THREE_PATH, BUCKET_THREE_QUOTA, + BUCKET_THREE_DATA_SIZE); + assertQuota(endpoint, BUCKET_FOUR_PATH, BUCKET_FOUR_QUOTA, + BUCKET_FOUR_DATA_SIZE); + + QuotaUsageResponse naKey = (QuotaUsageResponse) + endpoint.getQuotaUsage(KEY_PATH).getEntity(); + assertEquals(ResponseStatus.TYPE_NOT_APPLICABLE, naKey.getResponseCode()); + + QuotaUsageResponse invalid = (QuotaUsageResponse) + endpoint.getQuotaUsage(INVALID_PATH).getEntity(); + assertEquals(ResponseStatus.PATH_NOT_FOUND, invalid.getResponseCode()); + } + + private void assertQuota(NSSummaryEndpoint endpoint, String path, long quota, + long used) throws IOException { + QuotaUsageResponse res = + (QuotaUsageResponse) endpoint.getQuotaUsage(path).getEntity(); + assertEquals(quota, res.getQuota()); + assertEquals(used, res.getQuotaUsed()); + } + + @Override + void assertFileSizeDist(NSSummaryEndpoint endpoint) throws Exception { + NSSummaryEndpointTestBase.checkFileSizeDist(endpoint, ROOT_PATH, 2, 3, 3, 1); + NSSummaryEndpointTestBase.checkFileSizeDist(endpoint, VOL_PATH, 2, 1, 1, 1); + NSSummaryEndpointTestBase.checkFileSizeDist( + endpoint, BUCKET_ONE_PATH, 1, 1, 0, 1); + } + + @Override + void assertDataSizeUnderRootWithReplication(NSSummaryEndpoint endpoint) + throws IOException { + DUResponse res = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, ROOT_PATH, true); + assertEquals(ResponseStatus.OK, res.getStatus()); + assertEquals(REPLICA_UNDER_ROOT, res.getSizeWithReplica()); + assertEquals(REPLICA_UNDER_VOL, res.getDuData().get(0).getSizeWithReplica()); + } + + @Override + void assertDataSizeUnderVolWithReplication(NSSummaryEndpoint endpoint) + throws IOException { + DUResponse res = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, VOL_PATH, true); + assertEquals(ResponseStatus.OK, res.getStatus()); + assertEquals(REPLICA_UNDER_VOL, res.getSizeWithReplica()); + assertEquals(REPLICA_UNDER_BUCKET1, + res.getDuData().get(0).getSizeWithReplica()); + } + + void assertDataSizeUnderBucketOneWithReplication(NSSummaryEndpoint endpoint) + throws IOException { + DUResponse res = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, BUCKET_ONE_PATH, true); + assertEquals(ResponseStatus.OK, res.getStatus()); + assertEquals(REPLICA_UNDER_BUCKET1, res.getSizeWithReplica()); + } + + void assertDataSizeUnderBucketThreeWithReplication(NSSummaryEndpoint endpoint) + throws IOException { + DUResponse res = NSSummaryEndpointTestBase.getDiskUsage( + endpoint, BUCKET_THREE_PATH, true); + assertEquals(ResponseStatus.OK, res.getStatus()); + assertEquals(REPLICA_UNDER_BUCKET3, res.getSizeWithReplica()); + } + + @Override + void assertDataSizeUnderKeyWithReplication(NSSummaryEndpoint endpoint) + throws IOException { + DUResponse res = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, KEY_PATH, true); + assertEquals(ResponseStatus.OK, res.getStatus()); + assertEquals(REPLICA_UNDER_KEY, res.getSizeWithReplica()); + } + + void assertNormalizePathUptoBucket() { + assertEquals("/", OmUtils.normalizePathUptoBucket(null)); + assertEquals("/", OmUtils.normalizePathUptoBucket("")); + assertEquals("volume1/bucket1/key1/key2", + OmUtils.normalizePathUptoBucket("///volume1/bucket1/key1/key2")); + assertEquals("volume1/bucket1", + OmUtils.normalizePathUptoBucket("volume1/bucket1")); + assertEquals("volume1/bucket1/key1/key2", + OmUtils.normalizePathUptoBucket("volume1/bucket1/key1/key2")); + assertEquals("volume1/bucket1/key1//key2", + OmUtils.normalizePathUptoBucket("volume1/bucket1/key1//key2")); + assertEquals("volume/bucket/key$%#1/./////////key$%#2", + OmUtils.normalizePathUptoBucket("volume/bucket/key$%#1/./////////key$%#2")); + } + + @Override + void verifyConstructFullPath(ReconOMMetadataManager recon, + ReconNamespaceSummaryManager reconNamespaceSummaryManager) + throws IOException { + assertFullPath(reconNamespaceSummaryManager, KEY_TWO, VOL, BUCKET_ONE, + KEY_TWO_OBJECT_ID, "vol/bucket1/" + KEY_TWO); + assertFullPath(reconNamespaceSummaryManager, KEY_FIVE, VOL, BUCKET_TWO, + KEY_FIVE_OBJECT_ID, "vol/bucket2/" + KEY_FIVE); + assertFullPath(reconNamespaceSummaryManager, KEY_EIGHT, VOL_TWO, + BUCKET_THREE, KEY_EIGHT_OBJECT_ID, "vol2/bucket3/" + KEY_EIGHT); + assertFullPath(reconNamespaceSummaryManager, KEY_ELEVEN, VOL_TWO, + BUCKET_FOUR, KEY_ELEVEN_OBJECT_ID, "vol2/bucket4/" + KEY_ELEVEN); + } + + private void assertFullPath(ReconNamespaceSummaryManager nsMgr, + String keyName, String vol, String bucket, long objectId, + String expected) throws IOException { + OmKeyInfo keyInfo = new OmKeyInfo.Builder() + .setKeyName(keyName) + .setVolumeName(vol) + .setBucketName(bucket) + .setObjectID(objectId) + .build(); + assertEquals(expected, ReconUtils.constructFullPath(keyInfo, nsMgr)); + } +}
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/FsoNSSummaryScenario.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/FsoNSSummaryScenario.java new file mode 100644 index 0000000..ff93e6e --- /dev/null +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/FsoNSSummaryScenario.java
@@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api; + +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProviderWithFSO; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDirToOm; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.IOException; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; +import org.apache.hadoop.ozone.recon.ReconUtils; +import org.apache.hadoop.ozone.recon.api.types.NSSummary; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; +import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithFSO; + +/** + * FSO layout scenario. Directories live in the directory table and keys are + * parented by object ID. + */ +public class FsoNSSummaryScenario extends AbstractTreeNSSummaryScenario { + + public FsoNSSummaryScenario() { + super("FSO"); + } + + @Override + BucketLayout getBucketLayout() { + return BucketLayout.FILE_SYSTEM_OPTIMIZED; + } + + @Override + OzoneConfiguration newConfiguration() { + return new OzoneConfiguration(); + } + + @Override + void configureOmConfiguration(OzoneConfiguration conf) { + // FSO needs no extra OM configuration. + } + + @Override + OzoneManagerServiceProviderImpl mockOmServiceProvider() throws IOException { + return getMockOzoneManagerServiceProviderWithFSO(); + } + + @Override + void reprocess(ReconNamespaceSummaryManager reconNamespaceSummaryManager, + ReconOMMetadataManager reconOMMetadataManager, OzoneConfiguration conf) + throws IOException { + NSSummaryTaskWithFSO task = new NSSummaryTaskWithFSO( + reconNamespaceSummaryManager, reconOMMetadataManager, 10, 5, 20, 2000); + task.reprocessWithFSO(reconOMMetadataManager); + } + + @Override + @SuppressWarnings("checkstyle:ParameterNumber") + void writeDir(ReconOMMetadataManager recon, String dirName, String legacyPath, + long dirObjectId, long fsoParentObjectId, String bucketName, + String volName, long bucketObjectId, long volObjectId) + throws IOException { + writeDirToOm(recon, dirObjectId, fsoParentObjectId, bucketObjectId, + volObjectId, dirName); + } + + @Override + @SuppressWarnings("checkstyle:ParameterNumber") + void writeReplicatedKey(ReconOMMetadataManager recon, String key, + String bucket, String vol, String fileName, long keyObjectId, + long fsoParentObjectId, long bucketObjectId, long volObjectId, + OmKeyLocationInfoGroup locationGroup, long dataSize) throws IOException { + writeKeyToOm(recon, key, bucket, vol, fileName, keyObjectId, + fsoParentObjectId, bucketObjectId, volObjectId, + java.util.Collections.singletonList(locationGroup), getBucketLayout(), + dataSize, RatisReplicationConfig.getInstance(THREE)); + } + + @Override + void verifyConstructFullPath(ReconOMMetadataManager recon, + ReconNamespaceSummaryManager reconNamespaceSummaryManager) + throws IOException { + assertFullPath(reconNamespaceSummaryManager, FILE_TWO, VOL, BUCKET_ONE, + KEY_TWO_OBJECT_ID, DIR_TWO_OBJECT_ID, "vol/bucket1/dir1/dir2/file2"); + assertFullPath(reconNamespaceSummaryManager, FILE_THREE, VOL, BUCKET_ONE, + KEY_THREE_OBJECT_ID, DIR_THREE_OBJECT_ID, + "vol/bucket1/dir1/dir3/file3"); + assertFullPath(reconNamespaceSummaryManager, FILE_SIX, VOL, BUCKET_ONE, + KEY_SIX_OBJECT_ID, DIR_FOUR_OBJECT_ID, "vol/bucket1/dir1/dir4/file6"); + assertFullPath(reconNamespaceSummaryManager, FILE_ONE, VOL, BUCKET_ONE, + KEY_ONE_OBJECT_ID, BUCKET_ONE_OBJECT_ID, "vol/bucket1/file1"); + assertFullPath(reconNamespaceSummaryManager, FILE_NINE, VOL_TWO, + BUCKET_THREE, KEY_NINE_OBJECT_ID, DIR_FIVE_OBJECT_ID, + "vol2/bucket3/dir5/file9"); + + // When an NSSummary has parentId -1 (tree being rebuilt), constructFullPath + // returns an empty string. + NSSummary dir1Summary = + reconNamespaceSummaryManager.getNSSummary(DIR_ONE_OBJECT_ID); + dir1Summary.setParentId(-1); + reconNamespaceSummaryManager.deleteNSSummary(DIR_ONE_OBJECT_ID); + reconNamespaceSummaryManager.storeNSSummary(DIR_ONE_OBJECT_ID, dir1Summary); + assertEquals(-1, + reconNamespaceSummaryManager.getNSSummary(DIR_ONE_OBJECT_ID) + .getParentId(), + "The parentId should be updated to -1"); + + OmKeyInfo keyInfo = new OmKeyInfo.Builder() + .setKeyName(FILE_TWO) + .setVolumeName(VOL) + .setBucketName(BUCKET_ONE) + .setObjectID(KEY_TWO_OBJECT_ID) + .setParentObjectID(DIR_TWO_OBJECT_ID) + .build(); + assertEquals("", + ReconUtils.constructFullPath(keyInfo, reconNamespaceSummaryManager), + "Should return empty string when NSSummary tree is being rebuilt"); + } + + private void assertFullPath(ReconNamespaceSummaryManager nsMgr, + String keyName, String vol, String bucket, long objectId, + long parentObjectId, String expected) throws IOException { + OmKeyInfo keyInfo = new OmKeyInfo.Builder() + .setKeyName(keyName) + .setVolumeName(vol) + .setBucketName(bucket) + .setObjectID(objectId) + .setParentObjectID(parentObjectId) + .build(); + assertEquals(expected, ReconUtils.constructFullPath(keyInfo, nsMgr)); + } +}
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/LegacyNSSummaryScenario.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/LegacyNSSummaryScenario.java new file mode 100644 index 0000000..a38254e --- /dev/null +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/LegacyNSSummaryScenario.java
@@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api; + +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProvider; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.setConfiguration; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDirToOm; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.IOException; +import java.util.Collections; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; +import org.apache.hadoop.ozone.recon.ReconUtils; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; +import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithLegacy; + +/** + * Legacy layout scenario with {@code ozone.om.enable.filesystem.paths=true}, so + * the namespace tree matches FSO. Directories and keys are stored in the key + * table keyed by their full path; the object-ID parent is not used. + */ +public class LegacyNSSummaryScenario extends AbstractTreeNSSummaryScenario { + + private static final long PARENT_OBJECT_ID_ZERO = 0L; + + public LegacyNSSummaryScenario() { + super("LEGACY"); + } + + @Override + BucketLayout getBucketLayout() { + return BucketLayout.LEGACY; + } + + @Override + OzoneConfiguration newConfiguration() { + return new OzoneConfiguration(); + } + + @Override + void configureOmConfiguration(OzoneConfiguration conf) { + conf.set(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, "true"); + setConfiguration(conf); + } + + @Override + OzoneManagerServiceProviderImpl mockOmServiceProvider() throws IOException { + return getMockOzoneManagerServiceProvider(); + } + + @Override + void reprocess(ReconNamespaceSummaryManager reconNamespaceSummaryManager, + ReconOMMetadataManager reconOMMetadataManager, OzoneConfiguration conf) + throws IOException { + NSSummaryTaskWithLegacy task = new NSSummaryTaskWithLegacy( + reconNamespaceSummaryManager, reconOMMetadataManager, conf, 10); + task.reprocessWithLegacy(reconOMMetadataManager); + } + + @Override + @SuppressWarnings("checkstyle:ParameterNumber") + void writeDir(ReconOMMetadataManager recon, String dirName, String legacyPath, + long dirObjectId, long fsoParentObjectId, String bucketName, + String volName, long bucketObjectId, long volObjectId) + throws IOException { + writeDirToOm(recon, legacyPath, bucketName, volName, dirName, dirObjectId, + PARENT_OBJECT_ID_ZERO, bucketObjectId, volObjectId, getBucketLayout()); + } + + @Override + @SuppressWarnings("checkstyle:ParameterNumber") + void writeReplicatedKey(ReconOMMetadataManager recon, String key, + String bucket, String vol, String fileName, long keyObjectId, + long fsoParentObjectId, long bucketObjectId, long volObjectId, + OmKeyLocationInfoGroup locationGroup, long dataSize) throws IOException { + writeKeyToOm(recon, key, bucket, vol, fileName, keyObjectId, + PARENT_OBJECT_ID_ZERO, bucketObjectId, volObjectId, + Collections.singletonList(locationGroup), getBucketLayout(), dataSize, + RatisReplicationConfig.getInstance(THREE)); + } + + @Override + void verifyConstructFullPath(ReconOMMetadataManager recon, + ReconNamespaceSummaryManager reconNamespaceSummaryManager) + throws IOException { + // For key tables the parent object ID is not set; it defaults to -1 in the + // NSSummary, so full paths are reconstructed from the key name. + assertFullPath(reconNamespaceSummaryManager, "dir1/dir2/file2", + KEY_TWO_OBJECT_ID, "vol/bucket1/dir1/dir2/file2"); + assertFullPath(reconNamespaceSummaryManager, "dir1/dir2/", + DIR_TWO_OBJECT_ID, "vol/bucket1/dir1/dir2/"); + assertFullPath(reconNamespaceSummaryManager, "dir1/dir4/file6", + KEY_SIX_OBJECT_ID, "vol/bucket1/dir1/dir4/file6"); + } + + private void assertFullPath(ReconNamespaceSummaryManager nsMgr, + String keyName, long objectId, String expected) throws IOException { + OmKeyInfo keyInfo = new OmKeyInfo.Builder() + .setKeyName(keyName) + .setVolumeName(VOL) + .setBucketName(BUCKET_ONE) + .setObjectID(objectId) + .build(); + assertEquals(expected, ReconUtils.constructFullPath(keyInfo, nsMgr)); + } +}
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpointTestBase.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpointTestBase.java new file mode 100644 index 0000000..18a39ab --- /dev/null +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpointTestBase.java
@@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api; + +import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import javax.ws.rs.core.Response; +import org.apache.hadoop.hdds.client.BlockID; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State; +import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.container.ContainerManager; +import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; +import org.apache.hadoop.hdds.scm.container.ContainerReplica; +import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; +import org.apache.hadoop.ozone.recon.ReconConstants; +import org.apache.hadoop.ozone.recon.api.types.DUResponse; +import org.apache.hadoop.ozone.recon.api.types.EntityType; +import org.apache.hadoop.ozone.recon.api.types.FileSizeDistributionResponse; +import org.apache.hadoop.ozone.recon.api.types.KeyObjectDBInfo; +import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse; +import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; +import org.apache.hadoop.ozone.recon.scm.ReconNodeManager; +import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; + +/** + * Layout-independent scaffolding shared by the parameterized + * {@link TestNSSummaryEndpoint} suite. + * + * <p>Holds the mock SCM wiring, the block/container fixtures used to build + * multi-block keys, and the assertion helpers that are identical for every + * bucket layout. Everything that varies per layout (namespace tree, object + * IDs, expected sizes) lives in {@link NSSummaryTestScenario}. + */ +public abstract class NSSummaryEndpointTestBase { + + // container IDs backing the multi-block keys + static final long CONTAINER_ONE_ID = 1L; + static final long CONTAINER_TWO_ID = 2L; + static final long CONTAINER_THREE_ID = 3L; + static final long CONTAINER_FOUR_ID = 4L; + static final long CONTAINER_FIVE_ID = 5L; + static final long CONTAINER_SIX_ID = 6L; + + // container replica counts used only by the Container/Node/ClusterState + // endpoints. The NSSummary DU path derives sizeWithReplica from each key's + // own ReplicationConfig (OmKeyInfo#getReplicatedSize), so these counts do + // not influence any assertion in this suite; they exist purely so the mock + // ReconSCM binding is complete. + private static final int CONTAINER_ONE_REPLICA_COUNT = 3; + private static final int CONTAINER_TWO_REPLICA_COUNT = 2; + private static final int CONTAINER_THREE_REPLICA_COUNT = 4; + private static final int CONTAINER_FOUR_REPLICA_COUNT = 5; + private static final int CONTAINER_FIVE_REPLICA_COUNT = 2; + private static final int CONTAINER_SIX_REPLICA_COUNT = 3; + + // block lengths + private static final long BLOCK_ONE_LENGTH = 1000L; + private static final long BLOCK_TWO_LENGTH = 2000L; + private static final long BLOCK_THREE_LENGTH = 3000L; + private static final long BLOCK_FOUR_LENGTH = 4000L; + private static final long BLOCK_FIVE_LENGTH = 5000L; + private static final long BLOCK_SIX_LENGTH = 6000L; + + /** + * Location group over containers 1, 2 and 3. + */ + static OmKeyLocationInfoGroup getLocationInfoGroup1() { + List<OmKeyLocationInfo> locationInfoList = new ArrayList<>(); + locationInfoList.add(new OmKeyLocationInfo.Builder() + .setBlockID(new BlockID(CONTAINER_ONE_ID, 0L)) + .setLength(BLOCK_ONE_LENGTH) + .build()); + locationInfoList.add(new OmKeyLocationInfo.Builder() + .setBlockID(new BlockID(CONTAINER_TWO_ID, 0L)) + .setLength(BLOCK_TWO_LENGTH) + .build()); + locationInfoList.add(new OmKeyLocationInfo.Builder() + .setBlockID(new BlockID(CONTAINER_THREE_ID, 0L)) + .setLength(BLOCK_THREE_LENGTH) + .build()); + return new OmKeyLocationInfoGroup(0L, locationInfoList); + } + + /** + * Location group over containers 4, 5 and 6. + */ + static OmKeyLocationInfoGroup getLocationInfoGroup2() { + List<OmKeyLocationInfo> locationInfoList = new ArrayList<>(); + locationInfoList.add(new OmKeyLocationInfo.Builder() + .setBlockID(new BlockID(CONTAINER_FOUR_ID, 0L)) + .setLength(BLOCK_FOUR_LENGTH) + .build()); + locationInfoList.add(new OmKeyLocationInfo.Builder() + .setBlockID(new BlockID(CONTAINER_FIVE_ID, 0L)) + .setLength(BLOCK_FIVE_LENGTH) + .build()); + locationInfoList.add(new OmKeyLocationInfo.Builder() + .setBlockID(new BlockID(CONTAINER_SIX_ID, 0L)) + .setLength(BLOCK_SIX_LENGTH) + .build()); + return new OmKeyLocationInfoGroup(0L, locationInfoList); + } + + /** + * Generate a set of mock container replicas for a container. + * @param replicationFactor number of replicas + * @param containerID the container being replicated + * @return a set of container replicas for testing + */ + private static Set<ContainerReplica> generateMockContainerReplicas( + int replicationFactor, ContainerID containerID) { + Set<ContainerReplica> result = new HashSet<>(); + for (int i = 0; i < replicationFactor; ++i) { + DatanodeDetails randomDatanode = randomDatanodeDetails(); + result.add(new ContainerReplica.ContainerReplicaBuilder() + .setContainerID(containerID) + .setContainerState(State.OPEN) + .setDatanodeDetails(randomDatanode) + .build()); + } + return result; + } + + static ReconStorageContainerManagerFacade getMockReconSCM( + long rootQuota, long rootDataSize) throws ContainerNotFoundException { + ReconStorageContainerManagerFacade reconSCM = + mock(ReconStorageContainerManagerFacade.class); + ContainerManager containerManager = mock(ContainerManager.class); + + int[] replicaCounts = { + CONTAINER_ONE_REPLICA_COUNT, CONTAINER_TWO_REPLICA_COUNT, + CONTAINER_THREE_REPLICA_COUNT, CONTAINER_FOUR_REPLICA_COUNT, + CONTAINER_FIVE_REPLICA_COUNT, CONTAINER_SIX_REPLICA_COUNT}; + for (int i = 0; i < replicaCounts.length; i++) { + ContainerID containerID = ContainerID.valueOf(i + 1L); + when(containerManager.getContainerReplicas(containerID)).thenReturn( + generateMockContainerReplicas(replicaCounts[i], containerID)); + } + + when(reconSCM.getContainerManager()).thenReturn(containerManager); + ReconNodeManager mockReconNodeManager = mock(ReconNodeManager.class); + when(mockReconNodeManager.getStats()) + .thenReturn(getMockSCMRootStat(rootQuota, rootDataSize)); + when(reconSCM.getScmNodeManager()).thenReturn(mockReconNodeManager); + return reconSCM; + } + + private static SCMNodeStat getMockSCMRootStat(long rootQuota, + long rootDataSize) { + return new SCMNodeStat(rootQuota, rootDataSize, + rootQuota - rootDataSize, 0L, 0L, 0); + } + + static DUResponse getDiskUsage(NSSummaryEndpoint endpoint, String path, + boolean withReplica) throws IOException { + Response response = endpoint.getDiskUsage(path, false, withReplica, false); + return (DUResponse) response.getEntity(); + } + + static void checkFileSizeDist(NSSummaryEndpoint endpoint, String path, + int bin0, int bin1, int bin2, int bin3) throws Exception { + Response res = endpoint.getFileSizeDistribution(path); + FileSizeDistributionResponse fileSizeDistResObj = + (FileSizeDistributionResponse) res.getEntity(); + int[] fileSizeDist = fileSizeDistResObj.getFileSizeDist(); + assertEquals(bin0, fileSizeDist[0]); + assertEquals(bin1, fileSizeDist[1]); + assertEquals(bin2, fileSizeDist[2]); + assertEquals(bin3, fileSizeDist[3]); + for (int i = 4; i < ReconConstants.NUM_OF_FILE_SIZE_BINS; ++i) { + assertEquals(0, fileSizeDist[i]); + } + } + + static void assertBasicInfoNoPath(NSSummaryEndpoint endpoint, + String invalidPath) throws Exception { + Response invalidResponse = endpoint.getBasicInfo(invalidPath); + NamespaceSummaryResponse invalidObj = + (NamespaceSummaryResponse) invalidResponse.getEntity(); + assertEquals(ResponseStatus.PATH_NOT_FOUND, invalidObj.getStatus()); + assertNull(invalidObj.getCountStats()); + assertNull(invalidObj.getObjectDBInfo()); + } + + /** + * The key at {@code keyPath} (/vol/bucket2/file4) is written as a genuinely + * replicated (RATIS/THREE) multi-block key by every scenario, so the basic + * info must report the unreplicated data size and a RATIS replication type. + */ + static void assertBasicInfoKey(NSSummaryEndpoint endpoint, String keyPath, + long dataSize) throws Exception { + Response keyResponse = endpoint.getBasicInfo(keyPath); + NamespaceSummaryResponse keyResObj = + (NamespaceSummaryResponse) keyResponse.getEntity(); + assertEquals(EntityType.KEY, keyResObj.getEntityType()); + assertEquals("vol", + ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getVolumeName()); + assertEquals("bucket2", + ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getBucketName()); + assertEquals("file4", + ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getKeyName()); + assertEquals(dataSize, + ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getDataSize()); + assertEquals(HddsProtos.ReplicationType.RATIS, + ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()) + .getReplicationConfig().getReplicationType()); + } +}
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/NSSummaryTestScenario.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/NSSummaryTestScenario.java new file mode 100644 index 0000000..eafb35a --- /dev/null +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/NSSummaryTestScenario.java
@@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api; + +import java.io.File; +import java.io.IOException; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; + +/** + * A single parameterization of {@link TestNSSummaryEndpoint}: it owns the + * layout-specific fixture (config, OM provider mock, namespace population and + * reprocess) and the expected-value assertions for that layout. + * + * <p>Two concrete shapes exist: a tree fixture (FSO and Legacy with + * filesystem-paths enabled, see {@link AbstractTreeNSSummaryScenario}) and a + * flat fixture (OBS + Legacy with filesystem-paths disabled, see + * {@link FlatNSSummaryScenario}). Behaviour that only one shape supports is + * gated in the suite via the capability flags below and reached through + * {@code (AbstractTreeNSSummaryScenario) scenario} / + * {@code (FlatNSSummaryScenario) scenario} casts. + */ +public abstract class NSSummaryTestScenario { + + static final String TEST_USER = "TestUser"; + + // request paths shared by every layout + static final String ROOT_PATH = "/"; + static final String INVALID_PATH = "/vol/path/not/found"; + static final String KEY_PATH = "/vol/bucket2/file4"; + + // path-parsing fixture, identical across layouts + static final String TEST_PATH_UTILITY = "/vol1/buck1/a/b/c/d/e/file1.txt"; + static final String PARENT_DIR = "vol1/buck1/a/b/c/d/e"; + static final String[] TEST_NAMES = + new String[]{"vol1", "buck1", "a", "b", "c", "d", "e", "file1.txt"}; + static final String TEST_KEY_NAMES = "a/b/c/d/e/file1.txt"; + + private final String displayName; + + protected NSSummaryTestScenario(String displayName) { + this.displayName = displayName; + } + + // --------------------------------------------------------------------------- + // Fixture construction hooks + // --------------------------------------------------------------------------- + + abstract OzoneConfiguration newConfiguration(); + + abstract OzoneManagerServiceProviderImpl mockOmServiceProvider() + throws IOException; + + abstract OMMetadataManager initializeOmMetadataManager(File omDbDir, + OzoneConfiguration conf) throws IOException; + + /** + * Populate the Recon OM DB with the full namespace (directories and + * genuinely-replicated keys) and reprocess it into the NSSummary RocksDB. + */ + abstract void populateAndReprocess( + ReconNamespaceSummaryManager reconNamespaceSummaryManager, + ReconOMMetadataManager reconOMMetadataManager, + OzoneConfiguration conf) throws Exception; + + /** + * Write the extra single multi-block key exercised by + * {@code testDiskUsageWithReplication}. + */ + abstract void writeMultiBlockKey( + ReconOMMetadataManager reconOMMetadataManager) throws IOException; + + abstract void verifyConstructFullPath( + ReconOMMetadataManager reconOMMetadataManager, + ReconNamespaceSummaryManager reconNamespaceSummaryManager) + throws IOException; + + // Values needed to build the mock SCM before the fixture exists. + abstract long rootQuota(); + + abstract long rootDataSize(); + + // --------------------------------------------------------------------------- + // Capability flags used by the suite to gate structure-specific tests + // --------------------------------------------------------------------------- + + boolean hasDirectories() { + return false; + } + + boolean hasVolumeThree() { + return false; + } + + boolean isFlatLayout() { + return false; + } + + // --------------------------------------------------------------------------- + // Assertions common to every layout (expectations differ per scenario) + // --------------------------------------------------------------------------- + + abstract void assertBasicInfoRoot(NSSummaryEndpoint endpoint, + ReconOMMetadataManager reconOMMetadataManager) throws Exception; + + abstract void assertBasicInfoVolume(NSSummaryEndpoint endpoint) + throws Exception; + + abstract void assertBasicInfoBucketOne(NSSummaryEndpoint endpoint) + throws Exception; + + abstract void assertBasicInfoBucketTwo(NSSummaryEndpoint endpoint) + throws Exception; + + abstract void assertDiskUsageRoot(NSSummaryEndpoint endpoint) + throws Exception; + + abstract void assertDiskUsageVolume(NSSummaryEndpoint endpoint) + throws Exception; + + abstract void assertDiskUsageKey(NSSummaryEndpoint endpoint) throws Exception; + + abstract void assertQuotaUsage(NSSummaryEndpoint endpoint) throws Exception; + + abstract void assertFileSizeDist(NSSummaryEndpoint endpoint) throws Exception; + + /** Expected replicated size of the {@link #writeMultiBlockKey} key. */ + abstract long multiBlockKeyReplicatedSize(); + + abstract String multiBlockKeyPath(); + + abstract void assertDataSizeUnderRootWithReplication( + NSSummaryEndpoint endpoint) throws IOException; + + abstract void assertDataSizeUnderVolWithReplication( + NSSummaryEndpoint endpoint) throws IOException; + + abstract void assertDataSizeUnderKeyWithReplication( + NSSummaryEndpoint endpoint) throws IOException; + + @Override + public String toString() { + return displayName; + } +}
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/NSSummaryTests.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/NSSummaryTests.java deleted file mode 100644 index 9f5ff85..0000000 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/NSSummaryTests.java +++ /dev/null
@@ -1,195 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.recon.api; - -import static java.util.Collections.singletonList; -import static java.util.Collections.singletonMap; -import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS; -import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType.USER; -import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.WRITE; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; - -import javax.ws.rs.core.Response; -import org.apache.hadoop.hdds.protocol.StorageType; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.ozone.OzoneAcl; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo; -import org.apache.hadoop.ozone.recon.api.types.BucketObjectDBInfo; -import org.apache.hadoop.ozone.recon.api.types.EntityType; -import org.apache.hadoop.ozone.recon.api.types.KeyObjectDBInfo; -import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse; -import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; -import org.apache.hadoop.ozone.recon.api.types.VolumeObjectDBInfo; -import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; - -/** - * Shared NSSummary test cases. - */ -public abstract class NSSummaryTests { - private static final String ROOT_PATH = "/"; - private static final String VOL_PATH = "/vol"; - private static final String BUCKET_ONE_PATH = "/vol/bucket1"; - private static final String BUCKET_TWO_PATH = "/vol/bucket2"; - private static final String DIR_ONE_PATH = "/vol/bucket1/dir1"; - private static final String INVALID_PATH = "/vol/path/not/found"; - private static final String KEY_PATH = "/vol/bucket2/file4"; - - public static void testNSSummaryBasicInfoRoot( - NSSummaryEndpoint nsSummaryEndpoint, - ReconOMMetadataManager reconOMMetadataManager) throws Exception { - String username = "myuser"; - OmPrefixInfo omPrefixInfo = OmPrefixInfo.newBuilder() - .setName(ROOT_PATH) - .setObjectID(10) - .setUpdateID(100) - .setAcls(singletonList(OzoneAcl.of(USER, username, ACCESS, WRITE))) - .addAllMetadata(singletonMap("key", "value")) - .build(); - reconOMMetadataManager.getPrefixTable() - .put(OzoneConsts.OM_KEY_PREFIX, omPrefixInfo); - // Test root basics - Response rootResponse = nsSummaryEndpoint.getBasicInfo(ROOT_PATH); - NamespaceSummaryResponse rootResponseObj = - (NamespaceSummaryResponse) rootResponse.getEntity(); - assertEquals(EntityType.ROOT, rootResponseObj.getEntityType()); - assertEquals(3, rootResponseObj.getCountStats().getNumVolume()); - assertEquals(5, rootResponseObj.getCountStats().getNumBucket()); - assertEquals(7, rootResponseObj.getCountStats().getNumTotalDir()); - assertEquals(14, rootResponseObj.getCountStats().getNumTotalKey()); - assertEquals("USER", - rootResponseObj.getObjectDBInfo().getAcls().get(0).getType()); - assertEquals("WRITE", rootResponseObj.getObjectDBInfo().getAcls().get(0) - .getAclList().get(0)); - assertEquals(username, - rootResponseObj.getObjectDBInfo().getAcls().get(0).getName()); - assertEquals("value", - rootResponseObj.getObjectDBInfo().getMetadata().get("key")); - assertEquals("ACCESS", - rootResponseObj.getObjectDBInfo().getAcls().get(0).getScope()); - } - - public void testNSSummaryBasicInfoVolume( - NSSummaryEndpoint nsSummaryEndpoint) throws Exception { - Response volResponse = nsSummaryEndpoint.getBasicInfo(VOL_PATH); - NamespaceSummaryResponse volResponseObj = - (NamespaceSummaryResponse) volResponse.getEntity(); - assertEquals(EntityType.VOLUME, - volResponseObj.getEntityType()); - assertEquals(2, volResponseObj.getCountStats().getNumBucket()); - assertEquals(4, volResponseObj.getCountStats().getNumTotalDir()); - assertEquals(7, volResponseObj.getCountStats().getNumTotalKey()); - assertEquals("TestUser", ((VolumeObjectDBInfo) volResponseObj. - getObjectDBInfo()).getAdmin()); - assertEquals("TestUser", ((VolumeObjectDBInfo) volResponseObj. - getObjectDBInfo()).getOwner()); - assertEquals("vol", volResponseObj.getObjectDBInfo().getName()); - assertEquals(2097152, volResponseObj.getObjectDBInfo().getQuotaInBytes()); - assertEquals(-1, volResponseObj.getObjectDBInfo().getQuotaInNamespace()); - } - - public void testNSSummaryBasicInfoBucketOne(BucketLayout bucketLayout, - NSSummaryEndpoint nsSummaryEndpoint) throws Exception { - Response bucketOneResponse = - nsSummaryEndpoint.getBasicInfo(BUCKET_ONE_PATH); - NamespaceSummaryResponse bucketOneObj = - (NamespaceSummaryResponse) bucketOneResponse.getEntity(); - assertEquals(EntityType.BUCKET, bucketOneObj.getEntityType()); - assertEquals(4, bucketOneObj.getCountStats().getNumTotalDir()); - assertEquals(5, bucketOneObj.getCountStats().getNumTotalKey()); - assertEquals("vol", - ((BucketObjectDBInfo) bucketOneObj.getObjectDBInfo()).getVolumeName()); - assertEquals(StorageType.DISK, - ((BucketObjectDBInfo) - bucketOneObj.getObjectDBInfo()).getStorageType()); - assertEquals(bucketLayout, - ((BucketObjectDBInfo) - bucketOneObj.getObjectDBInfo()).getBucketLayout()); - assertEquals("bucket1", - ((BucketObjectDBInfo) bucketOneObj.getObjectDBInfo()).getName()); - } - - public void testNSSummaryBasicInfoBucketTwo( - BucketLayout bucketLayout, - NSSummaryEndpoint nsSummaryEndpoint) throws Exception { - Response bucketTwoResponse = - nsSummaryEndpoint.getBasicInfo(BUCKET_TWO_PATH); - NamespaceSummaryResponse bucketTwoObj = - (NamespaceSummaryResponse) bucketTwoResponse.getEntity(); - assertEquals(EntityType.BUCKET, bucketTwoObj.getEntityType()); - assertEquals(0, bucketTwoObj.getCountStats().getNumTotalDir()); - assertEquals(2, bucketTwoObj.getCountStats().getNumTotalKey()); - assertEquals("vol", - ((BucketObjectDBInfo) bucketTwoObj.getObjectDBInfo()).getVolumeName()); - assertEquals(StorageType.DISK, - ((BucketObjectDBInfo) - bucketTwoObj.getObjectDBInfo()).getStorageType()); - assertEquals(bucketLayout, - ((BucketObjectDBInfo) - bucketTwoObj.getObjectDBInfo()).getBucketLayout()); - assertEquals("bucket2", - ((BucketObjectDBInfo) bucketTwoObj.getObjectDBInfo()).getName()); - } - - public void testNSSummaryBasicInfoDir( - NSSummaryEndpoint nsSummaryEndpoint) throws Exception { - Response dirOneResponse = nsSummaryEndpoint.getBasicInfo(DIR_ONE_PATH); - NamespaceSummaryResponse dirOneObj = - (NamespaceSummaryResponse) dirOneResponse.getEntity(); - assertEquals(EntityType.DIRECTORY, dirOneObj.getEntityType()); - assertEquals(3, dirOneObj.getCountStats().getNumTotalDir()); - assertEquals(4, dirOneObj.getCountStats().getNumTotalKey()); - assertEquals("dir1", dirOneObj.getObjectDBInfo().getName()); - assertEquals(0, dirOneObj.getObjectDBInfo().getMetadata().size()); - assertEquals(0, dirOneObj.getObjectDBInfo().getQuotaInBytes()); - assertEquals(0, dirOneObj.getObjectDBInfo().getQuotaInNamespace()); - assertEquals(0, dirOneObj.getObjectDBInfo().getUsedNamespace()); - } - - public void testNSSummaryBasicInfoNoPath( - NSSummaryEndpoint nsSummaryEndpoint) throws Exception { - Response invalidResponse = nsSummaryEndpoint - .getBasicInfo(INVALID_PATH); - NamespaceSummaryResponse invalidObj = - (NamespaceSummaryResponse) invalidResponse.getEntity(); - assertEquals(ResponseStatus.PATH_NOT_FOUND, invalidObj.getStatus()); - assertNull(invalidObj.getCountStats()); - assertNull(invalidObj.getObjectDBInfo()); - } - - public void testNSSummaryBasicInfoKey( - NSSummaryEndpoint nsSummaryEndpoint) throws Exception { - Response keyResponse = nsSummaryEndpoint.getBasicInfo(KEY_PATH); - NamespaceSummaryResponse keyResObj = - (NamespaceSummaryResponse) keyResponse.getEntity(); - assertEquals(EntityType.KEY, keyResObj.getEntityType()); - assertEquals("vol", - ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getVolumeName()); - assertEquals("bucket2", - ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getBucketName()); - assertEquals("file4", - ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getKeyName()); - assertEquals(2049, - ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getDataSize()); - assertEquals(HddsProtos.ReplicationType.STAND_ALONE, - ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()). - getReplicationConfig().getReplicationType()); - } -}
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpoint.java new file mode 100644 index 0000000..e9e411f --- /dev/null +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpoint.java
@@ -0,0 +1,389 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api; + +import static org.apache.hadoop.ozone.OzoneConsts.KB; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; +import javax.ws.rs.core.Response; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.recon.ReconTestInjector; +import org.apache.hadoop.ozone.recon.ReconUtils; +import org.apache.hadoop.ozone.recon.api.handlers.BucketHandler; +import org.apache.hadoop.ozone.recon.api.handlers.EntityHandler; +import org.apache.hadoop.ozone.recon.api.types.DUResponse; +import org.apache.hadoop.ozone.recon.api.types.NSSummary; +import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; +import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; +import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.Parameter; +import org.junit.jupiter.params.ParameterizedClass; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Tests for the NSSummary REST APIs, run against every bucket layout. + * + * <p>Each {@link NSSummaryTestScenario} builds its own namespace and owns the + * expected values, so the tree (FSO, Legacy) and flat (OBS + Legacy) fixtures + * share this single suite. Structure-specific tests are gated with + * {@link org.junit.jupiter.api.Assumptions} on the scenario capability flags. + */ +@ParameterizedClass +@MethodSource("scenarios") +public class TestNSSummaryEndpoint extends NSSummaryEndpointTestBase { + + @TempDir + private Path temporaryFolder; + + @Parameter + private NSSummaryTestScenario scenario; + + private ReconOMMetadataManager reconOMMetadataManager; + private ReconNamespaceSummaryManager reconNamespaceSummaryManager; + private NSSummaryEndpoint nsSummaryEndpoint; + + static Stream<NSSummaryTestScenario> scenarios() { + return Stream.of( + new FsoNSSummaryScenario(), + new LegacyNSSummaryScenario(), + new FlatNSSummaryScenario()); + } + + @BeforeEach + public void setUp() throws Exception { + OzoneConfiguration conf = scenario.newConfiguration(); + OMMetadataManager omMetadataManager = scenario.initializeOmMetadataManager( + Files.createDirectory(temporaryFolder.resolve("JunitOmDBDir")).toFile(), + conf); + OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = + scenario.mockOmServiceProvider(); + reconOMMetadataManager = getTestReconOmMetadataManager(omMetadataManager, + Files.createDirectory(temporaryFolder.resolve("omMetadataDir")) + .toFile()); + + ReconTestInjector reconTestInjector = + new ReconTestInjector.Builder(temporaryFolder.toFile()) + .withReconOm(reconOMMetadataManager) + .withOmServiceProvider(ozoneManagerServiceProvider) + .withReconSqlDb() + .withContainerDB() + .addBinding(OzoneStorageContainerManager.class, + getMockReconSCM(scenario.rootQuota(), scenario.rootDataSize())) + .addBinding(StorageContainerServiceProvider.class, + mock(StorageContainerServiceProviderImpl.class)) + .addBinding(NSSummaryEndpoint.class) + .build(); + reconNamespaceSummaryManager = + reconTestInjector.getInstance(ReconNamespaceSummaryManager.class); + nsSummaryEndpoint = reconTestInjector.getInstance(NSSummaryEndpoint.class); + + scenario.populateAndReprocess(reconNamespaceSummaryManager, + reconOMMetadataManager, conf); + } + + // --------------------------------------------------------------------------- + // Tests common to every layout + // --------------------------------------------------------------------------- + + @Test + public void testUtility() { + String[] names = EntityHandler.parseRequestPath( + NSSummaryTestScenario.TEST_PATH_UTILITY); + assertArrayEquals(NSSummaryTestScenario.TEST_NAMES, names); + assertEquals(NSSummaryTestScenario.TEST_KEY_NAMES, + BucketHandler.getKeyName(names)); + assertEquals(NSSummaryTestScenario.TEST_PATH_UTILITY, + BucketHandler.buildSubpath(NSSummaryTestScenario.PARENT_DIR, + "file1.txt")); + } + + @Test + public void testGetBasicInfoRoot() throws Exception { + scenario.assertBasicInfoRoot(nsSummaryEndpoint, reconOMMetadataManager); + } + + @Test + public void testGetBasicInfoVol() throws Exception { + scenario.assertBasicInfoVolume(nsSummaryEndpoint); + } + + @Test + public void testGetBasicInfoBucketOne() throws Exception { + scenario.assertBasicInfoBucketOne(nsSummaryEndpoint); + } + + @Test + public void testGetBasicInfoBucketTwo() throws Exception { + scenario.assertBasicInfoBucketTwo(nsSummaryEndpoint); + } + + @Test + public void testGetBasicInfoNoPath() throws Exception { + assertBasicInfoNoPath(nsSummaryEndpoint, NSSummaryTestScenario.INVALID_PATH); + } + + @Test + public void testGetBasicInfoKey() throws Exception { + assertBasicInfoKey(nsSummaryEndpoint, NSSummaryTestScenario.KEY_PATH, + 2 * KB + 1); + } + + @Test + public void testDiskUsageRoot() throws Exception { + scenario.assertDiskUsageRoot(nsSummaryEndpoint); + } + + @Test + public void testDiskUsageVolume() throws Exception { + scenario.assertDiskUsageVolume(nsSummaryEndpoint); + } + + @Test + public void testDiskUsageKey() throws Exception { + scenario.assertDiskUsageKey(nsSummaryEndpoint); + } + + @Test + public void testDiskUsageUnknown() throws Exception { + Response invalidResponse = nsSummaryEndpoint.getDiskUsage( + NSSummaryTestScenario.INVALID_PATH, false, false, false); + DUResponse invalidObj = (DUResponse) invalidResponse.getEntity(); + assertEquals(ResponseStatus.PATH_NOT_FOUND, invalidObj.getStatus()); + } + + @Test + public void testDiskUsageWithReplication() throws Exception { + scenario.writeMultiBlockKey(reconOMMetadataManager); + DUResponse replicaDUResponse = getDiskUsage(nsSummaryEndpoint, + scenario.multiBlockKeyPath(), true); + assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); + assertEquals(scenario.multiBlockKeyReplicatedSize(), + replicaDUResponse.getSizeWithReplica()); + } + + @Test + public void testDataSizeUnderRootWithReplication() throws IOException { + scenario.assertDataSizeUnderRootWithReplication(nsSummaryEndpoint); + } + + @Test + public void testDataSizeUnderVolWithReplication() throws IOException { + scenario.assertDataSizeUnderVolWithReplication(nsSummaryEndpoint); + } + + @Test + public void testDataSizeUnderKeyWithReplication() throws IOException { + scenario.assertDataSizeUnderKeyWithReplication(nsSummaryEndpoint); + } + + @Test + public void testQuotaUsage() throws Exception { + scenario.assertQuotaUsage(nsSummaryEndpoint); + } + + @Test + public void testFileSizeDist() throws Exception { + scenario.assertFileSizeDist(nsSummaryEndpoint); + } + + @Test + public void testConstructFullPath() throws IOException { + scenario.verifyConstructFullPath(reconOMMetadataManager, + reconNamespaceSummaryManager); + } + + @Test + public void testConstructFullPathWithNegativeParentIdTriggersRebuild() + throws IOException { + long dirOneObjectId = 1L; + ReconNamespaceSummaryManager mockSummaryManager = + mock(ReconNamespaceSummaryManager.class); + NSSummary dir1Summary = new NSSummary(); + dir1Summary.setParentId(-1); + when(mockSummaryManager.getNSSummary(dirOneObjectId)) + .thenReturn(dir1Summary); + + OmKeyInfo keyInfo = new OmKeyInfo.Builder() + .setKeyName("file2") + .setVolumeName("vol") + .setBucketName("bucket1") + .setObjectID(2L) + .setParentObjectID(dirOneObjectId) + .build(); + + assertEquals("", + ReconUtils.constructFullPath(keyInfo, mockSummaryManager), + "Should return empty string when NSSummary has negative parentId"); + } + + // --------------------------------------------------------------------------- + // Tree-only tests (FSO, Legacy) + // --------------------------------------------------------------------------- + + @Test + public void testGetBasicInfoDir() throws Exception { + assumeTrue(scenario.hasDirectories()); + treeScenario().assertBasicInfoDir(nsSummaryEndpoint); + } + + @Test + public void testDiskUsageBucket() throws Exception { + assumeTrue(scenario.hasDirectories()); + treeScenario().assertDiskUsageBucket(nsSummaryEndpoint); + } + + @Test + public void testDiskUsageDir() throws Exception { + assumeTrue(scenario.hasDirectories()); + treeScenario().assertDiskUsageDir(nsSummaryEndpoint); + } + + @Test + public void testDataSizeUnderBucketWithReplication() throws IOException { + assumeTrue(scenario.hasDirectories()); + treeScenario().assertDataSizeUnderBucketWithReplication(nsSummaryEndpoint); + } + + @Test + public void testDataSizeUnderDirWithReplication() throws IOException { + assumeTrue(scenario.hasDirectories()); + treeScenario().assertDataSizeUnderDirWithReplication(nsSummaryEndpoint); + } + + @Test + public void testReplicatedSizePropagationUpwards() throws IOException { + assumeTrue(scenario.hasDirectories()); + treeScenario().assertReplicatedSizePropagation(nsSummaryEndpoint); + } + + @Test + public void testDataSizeUnderVolumeWithRatisReplication() throws IOException { + assumeTrue(scenario.hasVolumeThree()); + treeScenario().assertRatisReplicationUnderVolumeThree(nsSummaryEndpoint); + } + + @Test + public void testDataSizeUnderBucketWithRatisReplication() throws IOException { + assumeTrue(scenario.hasVolumeThree()); + treeScenario().assertRatisReplicationUnderBucketFive(nsSummaryEndpoint); + } + + @Test + public void testDataSizeUnderDirWithRatisReplication() throws IOException { + assumeTrue(scenario.hasVolumeThree()); + treeScenario().assertRatisReplicationUnderDirSix(nsSummaryEndpoint); + } + + // --------------------------------------------------------------------------- + // Flat-only tests (OBS + Legacy) + // --------------------------------------------------------------------------- + + @Test + public void testGetBasicInfoVolTwo() throws Exception { + assumeTrue(scenario.isFlatLayout()); + flatScenario().assertBasicInfoVolTwo(nsSummaryEndpoint); + } + + @Test + public void testGetBasicInfoBucketThree() throws Exception { + assumeTrue(scenario.isFlatLayout()); + flatScenario().assertBasicInfoBucketThree(nsSummaryEndpoint); + } + + @Test + public void testGetBasicInfoBucketFour() throws Exception { + assumeTrue(scenario.isFlatLayout()); + flatScenario().assertBasicInfoBucketFour(nsSummaryEndpoint); + } + + @Test + public void testDiskUsageVolTwo() throws Exception { + assumeTrue(scenario.isFlatLayout()); + flatScenario().assertDiskUsageVolTwo(nsSummaryEndpoint); + } + + @Test + public void testDiskUsageBucketOne() throws Exception { + assumeTrue(scenario.isFlatLayout()); + flatScenario().assertDiskUsageBucketOne(nsSummaryEndpoint); + } + + @Test + public void testDiskUsageBucketTwo() throws Exception { + assumeTrue(scenario.isFlatLayout()); + flatScenario().assertDiskUsageBucketTwo(nsSummaryEndpoint); + } + + @Test + public void testDiskUsageBucketThree() throws Exception { + assumeTrue(scenario.isFlatLayout()); + flatScenario().assertDiskUsageBucketThree(nsSummaryEndpoint); + } + + @Test + public void testDiskUsageKeys() throws Exception { + assumeTrue(scenario.isFlatLayout()); + flatScenario().assertDiskUsageKeys(nsSummaryEndpoint); + } + + @Test + public void testDataSizeUnderBucketOneWithReplication() throws IOException { + assumeTrue(scenario.isFlatLayout()); + flatScenario().assertDataSizeUnderBucketOneWithReplication( + nsSummaryEndpoint); + } + + @Test + public void testDataSizeUnderBucketThreeWithReplication() throws IOException { + assumeTrue(scenario.isFlatLayout()); + flatScenario().assertDataSizeUnderBucketThreeWithReplication( + nsSummaryEndpoint); + } + + @Test + public void testNormalizePathUptoBucket() { + assumeTrue(scenario.isFlatLayout()); + flatScenario().assertNormalizePathUptoBucket(); + } + + private AbstractTreeNSSummaryScenario treeScenario() { + return (AbstractTreeNSSummaryScenario) scenario; + } + + private FlatNSSummaryScenario flatScenario() { + return (FlatNSSummaryScenario) scenario; + } +}
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java deleted file mode 100644 index 320494b..0000000 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java +++ /dev/null
@@ -1,1577 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.recon.api; - -import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; -import static org.apache.hadoop.ozone.om.helpers.QuotaUtil.getReplicatedSize; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProviderWithFSO; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDirToOm; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; -import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import javax.ws.rs.core.Response; -import org.apache.hadoop.hdds.client.BlockID; -import org.apache.hadoop.hdds.client.RatisReplicationConfig; -import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State; -import org.apache.hadoop.hdds.scm.container.ContainerID; -import org.apache.hadoop.hdds.scm.container.ContainerManager; -import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; -import org.apache.hadoop.hdds.scm.container.ContainerReplica; -import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat; -import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; -import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; -import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; -import org.apache.hadoop.ozone.om.helpers.QuotaUtil; -import org.apache.hadoop.ozone.recon.ReconConstants; -import org.apache.hadoop.ozone.recon.ReconTestInjector; -import org.apache.hadoop.ozone.recon.ReconUtils; -import org.apache.hadoop.ozone.recon.api.handlers.BucketHandler; -import org.apache.hadoop.ozone.recon.api.handlers.EntityHandler; -import org.apache.hadoop.ozone.recon.api.types.DUResponse; -import org.apache.hadoop.ozone.recon.api.types.FileSizeDistributionResponse; -import org.apache.hadoop.ozone.recon.api.types.NSSummary; -import org.apache.hadoop.ozone.recon.api.types.QuotaUsageResponse; -import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; -import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; -import org.apache.hadoop.ozone.recon.scm.ReconNodeManager; -import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; -import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; -import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; -import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; -import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; -import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithFSO; -import org.apache.hadoop.util.Time; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -/** - * Test for NSSummary REST APIs with FSO. - * We tested on a mini file system with the following setting: - * vol - * / \ - * bucket1 bucket2 - * / \ / \ - * file1 dir1 file4 file5 - * / \ \ - * dir2 dir3 dir4 - * / \ \ - * file2 file3 file6 - * ---------------------------------------- - * vol2 - * / \ - * bucket3 bucket4 - * / \ / - * file8 dir5 file11 - * / \ - * file9 file10 - * ---------------------------------------- - * vol3 - * | - * bucket5 - * / \ - * file12 dir6 - * / \ - * file13 dir7 - * / - * file14 - * This is a test for the Rest APIs only. We have tested NSSummaryTask before, - * so there is no need to test process() on DB's updates - */ -public class TestNSSummaryEndpointWithFSO extends NSSummaryTests { - @TempDir - private Path temporaryFolder; - - private ReconOMMetadataManager reconOMMetadataManager; - private ReconNamespaceSummaryManager reconNamespaceSummaryManager; - private NSSummaryEndpoint nsSummaryEndpoint; - - private static final String TEST_PATH_UTILITY = - "/vol1/buck1/a/b/c/d/e/file1.txt"; - private static final String PARENT_DIR = "vol1/buck1/a/b/c/d/e"; - private static final String[] TEST_NAMES = - new String[]{"vol1", "buck1", "a", "b", "c", "d", "e", "file1.txt"}; - private static final String TEST_KEY_NAMES = "a/b/c/d/e/file1.txt"; - - // Object names in FSO-enabled format - private static final String VOL = "vol"; - private static final String VOL_TWO = "vol2"; - private static final String VOL_THREE = "vol3"; - private static final String BUCKET_ONE = "bucket1"; - private static final String BUCKET_TWO = "bucket2"; - private static final String BUCKET_THREE = "bucket3"; - private static final String BUCKET_FOUR = "bucket4"; - private static final String BUCKET_FIVE = "bucket5"; - private static final String KEY_ONE = "file1"; - private static final String KEY_TWO = "dir1/dir2/file2"; - private static final String KEY_THREE = "dir1/dir3/file3"; - private static final String KEY_FOUR = "file4"; - private static final String KEY_FIVE = "file5"; - private static final String KEY_SIX = "dir1/dir4/file6"; - private static final String KEY_SEVEN = "dir1/file7"; - private static final String KEY_EIGHT = "file8"; - private static final String KEY_NINE = "dir5/file9"; - private static final String KEY_TEN = "dir5/file10"; - private static final String KEY_ELEVEN = "file11"; - private static final String MULTI_BLOCK_KEY = "dir1/file7"; - private static final String MULTI_BLOCK_FILE = "file7"; - private static final String KEY_TWELVE = "file12"; - private static final String KEY_THIRTEEN = "dir6/file13"; - private static final String KEY_FOURTEEN = "dir6/dir7/file14"; - - private static final String FILE_ONE = "file1"; - private static final String FILE_TWO = "file2"; - private static final String FILE_THREE = "file3"; - private static final String FILE_FOUR = "file4"; - private static final String FILE_FIVE = "file5"; - private static final String FILE_SIX = "file6"; - private static final String FILE_SEVEN = "file7"; - private static final String FILE_EIGHT = "file8"; - private static final String FILE_NINE = "file9"; - private static final String FILE_TEN = "file10"; - private static final String FILE_ELEVEN = "file11"; - private static final String FILE_TWELVE = "file12"; - private static final String FILE_THIRTEEN = "file13"; - private static final String FILE_FOURTEEN = "file14"; - - private static final String DIR_ONE = "dir1"; - private static final String DIR_TWO = "dir2"; - private static final String DIR_THREE = "dir3"; - private static final String DIR_FOUR = "dir4"; - private static final String DIR_FIVE = "dir5"; - private static final String DIR_SIX = "dir6"; - private static final String DIR_SEVEN = "dir7"; - // objects IDs - private static final long VOL_OBJECT_ID = 0L; - private static final long BUCKET_ONE_OBJECT_ID = 1L; - private static final long BUCKET_TWO_OBJECT_ID = 2L; - private static final long KEY_ONE_OBJECT_ID = 3L; - private static final long DIR_ONE_OBJECT_ID = 4L; - private static final long KEY_TWO_OBJECT_ID = 5L; - private static final long KEY_FOUR_OBJECT_ID = 6L; - private static final long DIR_TWO_OBJECT_ID = 7L; - private static final long KEY_THREE_OBJECT_ID = 8L; - private static final long KEY_FIVE_OBJECT_ID = 9L; - private static final long KEY_SIX_OBJECT_ID = 10L; - private static final long DIR_THREE_OBJECT_ID = 11L; - private static final long DIR_FOUR_OBJECT_ID = 12L; - private static final long MULTI_BLOCK_KEY_OBJECT_ID = 13L; - private static final long KEY_SEVEN_OBJECT_ID = 13L; - private static final long VOL_TWO_OBJECT_ID = 14L; - private static final long BUCKET_THREE_OBJECT_ID = 15L; - private static final long BUCKET_FOUR_OBJECT_ID = 16L; - private static final long KEY_EIGHT_OBJECT_ID = 17L; - private static final long DIR_FIVE_OBJECT_ID = 18L; - private static final long KEY_NINE_OBJECT_ID = 19L; - private static final long KEY_TEN_OBJECT_ID = 20L; - private static final long KEY_ELEVEN_OBJECT_ID = 21L; - private static final long VOL_THREE_OBJECT_ID = 22L; - private static final long DIR_SIX_OBJECT_ID = 23L; - private static final long DIR_SEVEN_OBJECT_ID = 24L; - private static final long FILE_TWELVE_OBJECT_ID = 25L; - private static final long FILE_THIRTEEN_OBJECT_ID = 26L; - private static final long FILE_FOURTEEN_OBJECT_ID = 27L; - private static final long BUCKET_FIVE_OBJECT_ID = 28L; - - // container IDs - private static final long CONTAINER_ONE_ID = 1L; - private static final long CONTAINER_TWO_ID = 2L; - private static final long CONTAINER_THREE_ID = 3L; - private static final long CONTAINER_FOUR_ID = 4L; - private static final long CONTAINER_FIVE_ID = 5L; - private static final long CONTAINER_SIX_ID = 6L; - - // replication factors - private static final int CONTAINER_ONE_REPLICA_COUNT = 3; - private static final int CONTAINER_TWO_REPLICA_COUNT = 2; - private static final int CONTAINER_THREE_REPLICA_COUNT = 4; - private static final int CONTAINER_FOUR_REPLICA_COUNT = 5; - private static final int CONTAINER_FIVE_REPLICA_COUNT = 2; - private static final int CONTAINER_SIX_REPLICA_COUNT = 3; - - // block lengths - private static final long BLOCK_ONE_LENGTH = 1000L; - private static final long BLOCK_TWO_LENGTH = 2000L; - private static final long BLOCK_THREE_LENGTH = 3000L; - private static final long BLOCK_FOUR_LENGTH = 4000L; - private static final long BLOCK_FIVE_LENGTH = 5000L; - private static final long BLOCK_SIX_LENGTH = 6000L; - - // data size in bytes - private static final long KEY_ONE_SIZE = 500L; // bin 0 - private static final long KEY_TWO_SIZE = OzoneConsts.KB + 1; // bin 1 - private static final long KEY_THREE_SIZE = 4 * OzoneConsts.KB + 1; // bin 3 - private static final long KEY_FOUR_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 - private static final long KEY_FIVE_SIZE = 100L; // bin 0 - private static final long KEY_SIX_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 - private static final long KEY_SEVEN_SIZE = 4 * OzoneConsts.KB + 1; - private static final long KEY_EIGHT_SIZE = OzoneConsts.KB + 1; // bin 1 - private static final long KEY_NINE_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 - private static final long KEY_TEN_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 - private static final long KEY_ELEVEN_SIZE = OzoneConsts.KB + 1; // bin 1 - private static final long KEY_TWELVE_SIZE = OzoneConsts.KB; - private static final long KEY_THIRTEEN_SIZE = OzoneConsts.KB; - private static final long KEY_FOURTEEN_SIZE = OzoneConsts.KB; - - private static final long FILE1_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_ONE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE2_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_TWO_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE3_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_THREE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE4_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_FOUR_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE5_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_FIVE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE6_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_SIX_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE7_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_SEVEN_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE8_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_EIGHT_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE9_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_NINE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE10_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_TEN_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE11_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_ELEVEN_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE12_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_TWELVE_SIZE, RatisReplicationConfig.getInstance(THREE)); - private static final long FILE13_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_THIRTEEN_SIZE, RatisReplicationConfig.getInstance(THREE)); - private static final long FILE14_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_FOURTEEN_SIZE, RatisReplicationConfig.getInstance(THREE)); - private static final long MULTI_BLOCK_KEY_SIZE_WITH_REPLICA - = FILE7_SIZE_WITH_REPLICA; - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_ROOT - = FILE1_SIZE_WITH_REPLICA - + FILE2_SIZE_WITH_REPLICA - + FILE3_SIZE_WITH_REPLICA - + FILE4_SIZE_WITH_REPLICA - + FILE5_SIZE_WITH_REPLICA - + FILE6_SIZE_WITH_REPLICA - + FILE7_SIZE_WITH_REPLICA - + FILE8_SIZE_WITH_REPLICA - + FILE9_SIZE_WITH_REPLICA - + FILE10_SIZE_WITH_REPLICA - + FILE11_SIZE_WITH_REPLICA - + FILE12_SIZE_WITH_REPLICA - + FILE13_SIZE_WITH_REPLICA - + FILE14_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_VOL - = FILE1_SIZE_WITH_REPLICA - + FILE2_SIZE_WITH_REPLICA - + FILE3_SIZE_WITH_REPLICA - + FILE4_SIZE_WITH_REPLICA - + FILE5_SIZE_WITH_REPLICA - + FILE6_SIZE_WITH_REPLICA - + FILE7_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET1 - = FILE1_SIZE_WITH_REPLICA - + FILE2_SIZE_WITH_REPLICA - + FILE3_SIZE_WITH_REPLICA - + FILE6_SIZE_WITH_REPLICA - + FILE7_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR1 - = FILE2_SIZE_WITH_REPLICA - + FILE3_SIZE_WITH_REPLICA - + FILE6_SIZE_WITH_REPLICA - + FILE7_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR2 - = FILE2_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_KEY - = FILE4_SIZE_WITH_REPLICA; - - // quota in bytes - private static final long ROOT_QUOTA = 2 * (2 * OzoneConsts.MB); - private static final long VOL_QUOTA = 2 * OzoneConsts.MB; - private static final long VOL_TWO_QUOTA = 2 * OzoneConsts.MB; - private static final long VOL_THREE_QUOTA = 2 * OzoneConsts.MB; - private static final long BUCKET_ONE_QUOTA = OzoneConsts.MB; - private static final long BUCKET_TWO_QUOTA = OzoneConsts.MB; - private static final long BUCKET_THREE_QUOTA = OzoneConsts.MB; - private static final long BUCKET_FOUR_QUOTA = OzoneConsts.MB; - private static final long BUCKET_FIVE_QUOTA = OzoneConsts.MB; - - // mock client's path requests - private static final String TEST_USER = "TestUser"; - private static final String ROOT_PATH = "/"; - private static final String VOL_PATH = "/vol"; - private static final String VOL_TWO_PATH = "/vol2"; - private static final String BUCKET_ONE_PATH = "/vol/bucket1"; - private static final String BUCKET_TWO_PATH = "/vol/bucket2"; - private static final String DIR_ONE_PATH = "/vol/bucket1/dir1"; - private static final String DIR_TWO_PATH = "/vol/bucket1/dir1/dir2"; - private static final String DIR_THREE_PATH = "/vol/bucket1/dir1/dir3"; - private static final String DIR_FOUR_PATH = "/vol/bucket1/dir1/dir4"; - private static final String KEY_PATH = "/vol/bucket2/file4"; - private static final String MULTI_BLOCK_KEY_PATH = "/vol/bucket1/dir1/file7"; - private static final String INVALID_PATH = "/vol/path/not/found"; - private static final String VOL_THREE_PATH = "/vol3"; - - // some expected answers - private static final long ROOT_DATA_SIZE = KEY_ONE_SIZE + KEY_TWO_SIZE + - KEY_THREE_SIZE + KEY_FOUR_SIZE + KEY_FIVE_SIZE + KEY_SIX_SIZE + KEY_SEVEN_SIZE + - KEY_EIGHT_SIZE + KEY_NINE_SIZE + KEY_TEN_SIZE + KEY_ELEVEN_SIZE + - FILE12_SIZE_WITH_REPLICA + FILE13_SIZE_WITH_REPLICA + FILE14_SIZE_WITH_REPLICA; - - private static final long VOL_DATA_SIZE = KEY_ONE_SIZE + KEY_TWO_SIZE + - KEY_THREE_SIZE + KEY_FOUR_SIZE + KEY_FIVE_SIZE + KEY_SIX_SIZE + KEY_SEVEN_SIZE; - - private static final long VOL_TWO_DATA_SIZE = - KEY_EIGHT_SIZE + KEY_NINE_SIZE + KEY_TEN_SIZE + KEY_ELEVEN_SIZE; - - private static final long BUCKET_ONE_DATA_SIZE = KEY_ONE_SIZE + KEY_TWO_SIZE + - KEY_THREE_SIZE + KEY_SIX_SIZE + KEY_SEVEN_SIZE; - - private static final long BUCKET_TWO_DATA_SIZE = - KEY_FOUR_SIZE + KEY_FIVE_SIZE; - - private static final long DIR_ONE_DATA_SIZE = KEY_TWO_SIZE + - KEY_THREE_SIZE + KEY_SIX_SIZE + KEY_SEVEN_SIZE; - - @BeforeEach - public void setUp() throws Exception { - OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); - ozoneConfiguration.setLong(OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD, - 10); - OMMetadataManager omMetadataManager = initializeNewOmMetadataManager( - Files.createDirectory(temporaryFolder.resolve("JunitOmDBDir")) - .toFile()); - OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = - getMockOzoneManagerServiceProviderWithFSO(); - reconOMMetadataManager = getTestReconOmMetadataManager(omMetadataManager, - Files.createDirectory(temporaryFolder.resolve("OmMetataDir")).toFile()); - - ReconTestInjector reconTestInjector = - new ReconTestInjector.Builder(temporaryFolder.toFile()) - .withReconOm(reconOMMetadataManager) - .withOmServiceProvider(ozoneManagerServiceProvider) - .withReconSqlDb() - .withContainerDB() - .addBinding(OzoneStorageContainerManager.class, - getMockReconSCM()) - .addBinding(StorageContainerServiceProvider.class, - mock(StorageContainerServiceProviderImpl.class)) - .addBinding(NSSummaryEndpoint.class) - .build(); - this.reconNamespaceSummaryManager = - reconTestInjector.getInstance(ReconNamespaceSummaryManager.class); - nsSummaryEndpoint = reconTestInjector.getInstance(NSSummaryEndpoint.class); - - // populate OM DB and reprocess into Recon RocksDB - populateOMDB(); - populateVolumeThree(); - setUpMultiBlockReplicatedKeys(); - NSSummaryTaskWithFSO nSSummaryTaskWithFso = - new NSSummaryTaskWithFSO(reconNamespaceSummaryManager, reconOMMetadataManager, 10, 5, 20, 2000); - nSSummaryTaskWithFso.reprocessWithFSO(reconOMMetadataManager); - } - - @Test - public void testUtility() { - String[] names = EntityHandler.parseRequestPath(TEST_PATH_UTILITY); - assertArrayEquals(TEST_NAMES, names); - String keyName = BucketHandler.getKeyName(names); - assertEquals(TEST_KEY_NAMES, keyName); - String subpath = BucketHandler.buildSubpath(PARENT_DIR, "file1.txt"); - assertEquals(TEST_PATH_UTILITY, subpath); - } - - @Test - public void testGetBasicInfoRoot() throws Exception { - // Test root basics - testNSSummaryBasicInfoRoot( - nsSummaryEndpoint, reconOMMetadataManager); - } - - @Test - public void testGetBasicInfoVol() throws Exception { - // Test volume basics - testNSSummaryBasicInfoVolume(nsSummaryEndpoint); - } - - @Test - public void testGetBasicInfoBucketOne() throws Exception { - // Test bucket 1's basics - testNSSummaryBasicInfoBucketOne( - BucketLayout.FILE_SYSTEM_OPTIMIZED, - nsSummaryEndpoint); - } - - @Test - public void testGetBasicInfoBucketTwo() throws Exception { - // Test bucket 2's basics - testNSSummaryBasicInfoBucketTwo( - BucketLayout.FILE_SYSTEM_OPTIMIZED, - nsSummaryEndpoint); - } - - @Test - public void testGetBasicInfoDir() throws Exception { - // Test intermediate directory basics - testNSSummaryBasicInfoDir(nsSummaryEndpoint); - } - - @Test - public void testGetBasicInfoNoPath() throws Exception { - // Test invalid path - testNSSummaryBasicInfoNoPath(nsSummaryEndpoint); - } - - @Test - public void testGetBasicInfoKey() throws Exception { - // Test key - testNSSummaryBasicInfoKey(nsSummaryEndpoint); - } - - @Test - public void testDiskUsageRoot() throws Exception { - // root level DU - Response rootResponse = nsSummaryEndpoint.getDiskUsage(ROOT_PATH, - false, false, false); - DUResponse duRootRes = (DUResponse) rootResponse.getEntity(); - assertEquals(3, duRootRes.getCount()); - List<DUResponse.DiskUsage> duRootData = duRootRes.getDuData(); - // sort based on subpath - Collections.sort(duRootData, - Comparator.comparing(DUResponse.DiskUsage::getSubpath)); - DUResponse.DiskUsage duVol1 = duRootData.get(0); - DUResponse.DiskUsage duVol2 = duRootData.get(1); - assertEquals(VOL_PATH, duVol1.getSubpath()); - assertEquals(VOL_TWO_PATH, duVol2.getSubpath()); - assertEquals(VOL_DATA_SIZE, duVol1.getSize()); - assertEquals(VOL_TWO_DATA_SIZE, duVol2.getSize()); - } - - @Test - public void testDiskUsageVolume() throws Exception { - // volume level DU - Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, - false, false, false); - DUResponse duVolRes = (DUResponse) volResponse.getEntity(); - assertEquals(2, duVolRes.getCount()); - List<DUResponse.DiskUsage> duData = duVolRes.getDuData(); - // sort based on subpath - Collections.sort(duData, - Comparator.comparing(DUResponse.DiskUsage::getSubpath)); - DUResponse.DiskUsage duBucket1 = duData.get(0); - DUResponse.DiskUsage duBucket2 = duData.get(1); - assertEquals(BUCKET_ONE_PATH, duBucket1.getSubpath()); - assertEquals(BUCKET_TWO_PATH, duBucket2.getSubpath()); - assertEquals(BUCKET_ONE_DATA_SIZE, duBucket1.getSize()); - assertEquals(BUCKET_TWO_DATA_SIZE, duBucket2.getSize()); - - } - - @Test - public void testDiskUsageBucket() throws Exception { - // bucket level DU - Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, - false, false, false); - DUResponse duBucketResponse = (DUResponse) bucketResponse.getEntity(); - assertEquals(1, duBucketResponse.getCount()); - DUResponse.DiskUsage duDir1 = duBucketResponse.getDuData().get(0); - assertEquals(DIR_ONE_PATH, duDir1.getSubpath()); - assertEquals(DIR_ONE_DATA_SIZE, duDir1.getSize()); - - } - - @Test - public void testDiskUsageDir() throws Exception { - // dir level DU - Response dirResponse = nsSummaryEndpoint.getDiskUsage(DIR_ONE_PATH, - false, false, false); - DUResponse duDirReponse = (DUResponse) dirResponse.getEntity(); - assertEquals(3, duDirReponse.getCount()); - List<DUResponse.DiskUsage> duSubDir = duDirReponse.getDuData(); - Collections.sort(duSubDir, - Comparator.comparing(DUResponse.DiskUsage::getSubpath)); - DUResponse.DiskUsage duDir2 = duSubDir.get(0); - DUResponse.DiskUsage duDir3 = duSubDir.get(1); - DUResponse.DiskUsage duDir4 = duSubDir.get(2); - assertEquals(DIR_TWO_PATH, duDir2.getSubpath()); - assertEquals(KEY_TWO_SIZE, duDir2.getSize()); - - assertEquals(DIR_THREE_PATH, duDir3.getSubpath()); - assertEquals(KEY_THREE_SIZE, duDir3.getSize()); - - assertEquals(DIR_FOUR_PATH, duDir4.getSubpath()); - assertEquals(KEY_SIX_SIZE, duDir4.getSize()); - - } - - @Test - public void testDiskUsageKey() throws Exception { - // key level DU - Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_PATH, - false, false, false); - DUResponse keyObj = (DUResponse) keyResponse.getEntity(); - assertEquals(0, keyObj.getCount()); - assertEquals(KEY_FOUR_SIZE, keyObj.getSize()); - - } - - @Test - public void testDiskUsageUnknown() throws Exception { - // invalid path check - Response invalidResponse = nsSummaryEndpoint.getDiskUsage(INVALID_PATH, - false, false, false); - DUResponse invalidObj = (DUResponse) invalidResponse.getEntity(); - assertEquals(ResponseStatus.PATH_NOT_FOUND, - invalidObj.getStatus()); - } - - @Test - public void testDiskUsageWithReplication() throws Exception { - setUpMultiBlockKey(); - Response keyResponse = nsSummaryEndpoint.getDiskUsage(MULTI_BLOCK_KEY_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_KEY_SIZE_WITH_REPLICA, - replicaDUResponse.getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderRootWithReplication() throws IOException { - // withReplica is true - Response rootResponse = nsSummaryEndpoint.getDiskUsage(ROOT_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) rootResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_ROOT, - replicaDUResponse.getSizeWithReplica()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_VOL, - replicaDUResponse.getDuData().get(0).getSizeWithReplica()); - - } - - @Test - public void testDataSizeUnderVolWithReplication() throws IOException { - Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) volResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_VOL, - replicaDUResponse.getSizeWithReplica()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET1, - replicaDUResponse.getDuData().get(0).getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderBucketWithReplication() throws IOException { - Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) bucketResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET1, - replicaDUResponse.getSizeWithReplica()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR1, - replicaDUResponse.getDuData().get(0).getSizeWithReplica()); - } - - @Test - public void testReplicatedSizePropagationUpwards() throws IOException { - // Test that replicated size propagates correctly from files up through the directory hierarchy - - // Get disk usage for individual files first to establish baseline - DUResponse file2Response = getDiskUsageResponse(DIR_TWO_PATH + "/file2"); - DUResponse file3Response = getDiskUsageResponse(DIR_THREE_PATH + "/file3"); - DUResponse file6Response = getDiskUsageResponse(DIR_FOUR_PATH + "/file6"); - DUResponse file7Response = getDiskUsageResponse(DIR_ONE_PATH + "/file7"); - - // Verify individual file replicated sizes - assertEquals(FILE2_SIZE_WITH_REPLICA, file2Response.getSizeWithReplica()); - assertEquals(FILE3_SIZE_WITH_REPLICA, file3Response.getSizeWithReplica()); - assertEquals(FILE6_SIZE_WITH_REPLICA, file6Response.getSizeWithReplica()); - assertEquals(FILE7_SIZE_WITH_REPLICA, file7Response.getSizeWithReplica()); - - // Test dir2 (contains only file2) - DUResponse dir2Response = getDiskUsageResponse(DIR_TWO_PATH); - assertEquals(FILE2_SIZE_WITH_REPLICA, dir2Response.getSizeWithReplica()); - - // Test dir3 (contains only file3) - DUResponse dir3Response = getDiskUsageResponse(DIR_THREE_PATH); - assertEquals(FILE3_SIZE_WITH_REPLICA, dir3Response.getSizeWithReplica()); - - // Test dir4 (contains only file6) - DUResponse dir4Response = getDiskUsageResponse(DIR_FOUR_PATH); - assertEquals(FILE6_SIZE_WITH_REPLICA, dir4Response.getSizeWithReplica()); - - // Test dir1 (contains file7 directly + dir2, dir3, dir4 contents) - DUResponse dir1Response = getDiskUsageResponse(DIR_ONE_PATH); - long expectedDir1ReplicatedSize = FILE2_SIZE_WITH_REPLICA + FILE3_SIZE_WITH_REPLICA + - FILE6_SIZE_WITH_REPLICA + FILE7_SIZE_WITH_REPLICA; - assertEquals(expectedDir1ReplicatedSize, dir1Response.getSizeWithReplica()); - - // Test bucket1 (contains file1 directly + all dir1 contents) - DUResponse bucket1Response = getDiskUsageResponse(BUCKET_ONE_PATH); - long expectedBucket1ReplicatedSize = FILE1_SIZE_WITH_REPLICA + expectedDir1ReplicatedSize; - assertEquals(expectedBucket1ReplicatedSize, bucket1Response.getSizeWithReplica()); - - // Test bucket2 (contains file4 and file5) - DUResponse bucket2Response = getDiskUsageResponse(BUCKET_TWO_PATH); - long expectedBucket2ReplicatedSize = FILE4_SIZE_WITH_REPLICA + FILE5_SIZE_WITH_REPLICA; - assertEquals(expectedBucket2ReplicatedSize, bucket2Response.getSizeWithReplica()); - - // Test vol (contains bucket1 + bucket2) - DUResponse volResponse = getDiskUsageResponse(VOL_PATH); - long expectedVolReplicatedSize = expectedBucket1ReplicatedSize + expectedBucket2ReplicatedSize; - assertEquals(expectedVolReplicatedSize, volResponse.getSizeWithReplica()); - - // Test root (contains vol + vol2) - DUResponse rootResponse = getDiskUsageResponse(ROOT_PATH); - long expectedVol2ReplicatedSize = FILE8_SIZE_WITH_REPLICA + FILE9_SIZE_WITH_REPLICA + - FILE10_SIZE_WITH_REPLICA + FILE11_SIZE_WITH_REPLICA; - long vol3TotalSize = FILE12_SIZE_WITH_REPLICA + FILE13_SIZE_WITH_REPLICA + FILE14_SIZE_WITH_REPLICA; - long expectedRootReplicatedSize = expectedVolReplicatedSize + expectedVol2ReplicatedSize + vol3TotalSize; - assertEquals(expectedRootReplicatedSize, rootResponse.getSizeWithReplica()); - } - - private DUResponse getDiskUsageResponse(String path) throws IOException { - Response response = nsSummaryEndpoint.getDiskUsage(path, false, true, false); - return (DUResponse) response.getEntity(); - } - - /** - * When calculating DU under dir1 - * there are 3 keys, file2, file3, file6. - * There is one direct key, file7. - * @throws IOException - */ - @Test - public void testDataSizeUnderDirWithReplication() throws IOException { - Response dir1Response = nsSummaryEndpoint.getDiskUsage(DIR_ONE_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) dir1Response.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR1, - replicaDUResponse.getSizeWithReplica()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR2, - replicaDUResponse.getDuData().get(0).getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderKeyWithReplication() throws IOException { - Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_KEY, - replicaDUResponse.getSizeWithReplica()); - } - - @Test - public void testQuotaUsage() throws Exception { - // root level quota usage - Response rootResponse = nsSummaryEndpoint.getQuotaUsage(ROOT_PATH); - QuotaUsageResponse quRootRes = - (QuotaUsageResponse) rootResponse.getEntity(); - assertEquals(ROOT_QUOTA, quRootRes.getQuota()); - assertEquals(ROOT_DATA_SIZE, quRootRes.getQuotaUsed()); - - // volume level quota usage - Response volResponse = nsSummaryEndpoint.getQuotaUsage(VOL_PATH); - QuotaUsageResponse quVolRes = (QuotaUsageResponse) volResponse.getEntity(); - assertEquals(VOL_QUOTA, quVolRes.getQuota()); - assertEquals(VOL_DATA_SIZE, quVolRes.getQuotaUsed()); - - // bucket level quota usage - Response bucketRes = nsSummaryEndpoint.getQuotaUsage(BUCKET_ONE_PATH); - QuotaUsageResponse quBucketRes = (QuotaUsageResponse) bucketRes.getEntity(); - assertEquals(BUCKET_ONE_QUOTA, quBucketRes.getQuota()); - assertEquals(BUCKET_ONE_DATA_SIZE, quBucketRes.getQuotaUsed()); - - Response bucketRes2 = nsSummaryEndpoint.getQuotaUsage(BUCKET_TWO_PATH); - QuotaUsageResponse quBucketRes2 = - (QuotaUsageResponse) bucketRes2.getEntity(); - assertEquals(BUCKET_TWO_QUOTA, quBucketRes2.getQuota()); - assertEquals(BUCKET_TWO_DATA_SIZE, quBucketRes2.getQuotaUsed()); - - // other level not applicable - Response naResponse1 = nsSummaryEndpoint.getQuotaUsage(DIR_ONE_PATH); - QuotaUsageResponse quotaUsageResponse1 = - (QuotaUsageResponse) naResponse1.getEntity(); - assertEquals(ResponseStatus.TYPE_NOT_APPLICABLE, - quotaUsageResponse1.getResponseCode()); - - Response naResponse2 = nsSummaryEndpoint.getQuotaUsage(KEY_PATH); - QuotaUsageResponse quotaUsageResponse2 = - (QuotaUsageResponse) naResponse2.getEntity(); - assertEquals(ResponseStatus.TYPE_NOT_APPLICABLE, - quotaUsageResponse2.getResponseCode()); - - // invalid path request - Response invalidRes = nsSummaryEndpoint.getQuotaUsage(INVALID_PATH); - QuotaUsageResponse invalidResObj = - (QuotaUsageResponse) invalidRes.getEntity(); - assertEquals(ResponseStatus.PATH_NOT_FOUND, - invalidResObj.getResponseCode()); - } - - @Test - public void testFileSizeDist() throws Exception { - checkFileSizeDist(ROOT_PATH, 5, 3, 4, 2); - checkFileSizeDist(VOL_PATH, 2, 1, 2, 2); - checkFileSizeDist(BUCKET_ONE_PATH, 1, 1, 1, 2); - checkFileSizeDist(DIR_ONE_PATH, 0, 1, 1, 2); - } - - public void checkFileSizeDist(String path, int bin0, - int bin1, int bin2, int bin3) throws Exception { - Response res = nsSummaryEndpoint.getFileSizeDistribution(path); - FileSizeDistributionResponse fileSizeDistResObj = - (FileSizeDistributionResponse) res.getEntity(); - int[] fileSizeDist = fileSizeDistResObj.getFileSizeDist(); - assertEquals(bin0, fileSizeDist[0]); - assertEquals(bin1, fileSizeDist[1]); - assertEquals(bin2, fileSizeDist[2]); - assertEquals(bin3, fileSizeDist[3]); - for (int i = 4; i < ReconConstants.NUM_OF_FILE_SIZE_BINS; ++i) { - assertEquals(0, fileSizeDist[i]); - } - } - - @Test - public void testConstructFullPath() throws IOException { - OmKeyInfo keyInfo = new OmKeyInfo.Builder() - .setKeyName("file2") - .setVolumeName(VOL) - .setBucketName(BUCKET_ONE) - .setObjectID(KEY_TWO_OBJECT_ID) - .setParentObjectID(DIR_TWO_OBJECT_ID) - .build(); - // Call constructFullPath and verify the result - String fullPath = ReconUtils.constructFullPath(keyInfo, - reconNamespaceSummaryManager); - String expectedPath = "vol/bucket1/dir1/dir2/file2"; - Assertions.assertEquals(expectedPath, fullPath); - - // Create key info for file 3 - keyInfo = new OmKeyInfo.Builder() - .setKeyName("file3") - .setVolumeName(VOL) - .setBucketName(BUCKET_ONE) - .setObjectID(KEY_THREE_OBJECT_ID) - .setParentObjectID(DIR_THREE_OBJECT_ID) - .build(); - fullPath = ReconUtils.constructFullPath(keyInfo, - reconNamespaceSummaryManager); - expectedPath = "vol/bucket1/dir1/dir3/file3"; - Assertions.assertEquals(expectedPath, fullPath); - - // Create key info for file 6 - keyInfo = new OmKeyInfo.Builder() - .setKeyName("file6") - .setVolumeName(VOL) - .setBucketName(BUCKET_ONE) - .setObjectID(KEY_SIX_OBJECT_ID) - .setParentObjectID(DIR_FOUR_OBJECT_ID) - .build(); - fullPath = ReconUtils.constructFullPath(keyInfo, - reconNamespaceSummaryManager); - expectedPath = "vol/bucket1/dir1/dir4/file6"; - Assertions.assertEquals(expectedPath, fullPath); - - // Create key info for file 1 - keyInfo = new OmKeyInfo.Builder() - .setKeyName("file1") - .setVolumeName(VOL) - .setBucketName(BUCKET_ONE) - .setObjectID(KEY_ONE_OBJECT_ID) - .setParentObjectID(BUCKET_ONE_OBJECT_ID) - .build(); - fullPath = ReconUtils.constructFullPath(keyInfo, - reconNamespaceSummaryManager); - expectedPath = "vol/bucket1/file1"; - Assertions.assertEquals(expectedPath, fullPath); - - // Create key info for file 9 - keyInfo = new OmKeyInfo.Builder() - .setKeyName("file9") - .setVolumeName(VOL_TWO) - .setBucketName(BUCKET_THREE) - .setObjectID(KEY_NINE_OBJECT_ID) - .setParentObjectID(DIR_FIVE_OBJECT_ID) - .build(); - fullPath = ReconUtils.constructFullPath(keyInfo, - reconNamespaceSummaryManager); - expectedPath = "vol2/bucket3/dir5/file9"; - Assertions.assertEquals(expectedPath, fullPath); - - // Check for when we encounter a NSSUmamry with parentId -1 - // Fetch NSSummary for dir1 and immediately update its parentId. - NSSummary dir1Summary = reconNamespaceSummaryManager.getNSSummary(DIR_ONE_OBJECT_ID); - dir1Summary.setParentId(-1); // Update parentId to -1 - - reconNamespaceSummaryManager.deleteNSSummary(DIR_ONE_OBJECT_ID); - reconNamespaceSummaryManager.storeNSSummary(DIR_ONE_OBJECT_ID, dir1Summary); - - NSSummary changedDir1Summary = reconNamespaceSummaryManager.getNSSummary(DIR_ONE_OBJECT_ID); - Assertions.assertEquals(-1, changedDir1Summary.getParentId(), "The parentId should be updated to -1"); - - keyInfo = new OmKeyInfo.Builder() - .setKeyName("file2") - .setVolumeName(VOL) - .setBucketName(BUCKET_ONE) - .setObjectID(KEY_TWO_OBJECT_ID) - .setParentObjectID(DIR_TWO_OBJECT_ID) - .build(); - // Call constructFullPath and verify the result - should return empty string when NSSummary parent is invalid - fullPath = ReconUtils.constructFullPath(keyInfo, reconNamespaceSummaryManager); - Assertions.assertEquals("", fullPath, "Should return empty string when NSSummary tree is being rebuilt"); - } - - @Test - public void testConstructFullPathWithNegativeParentIdTriggersRebuild() throws IOException { - // Setup - long dirOneObjectId = 1L; // Sample object ID for the directory - ReconNamespaceSummaryManager mockSummaryManager = mock(ReconNamespaceSummaryManager.class); - NSSummary dir1Summary = new NSSummary(); - dir1Summary.setParentId(-1); // Simulate directory at the top of the tree - when(mockSummaryManager.getNSSummary(dirOneObjectId)).thenReturn(dir1Summary); - - OmKeyInfo keyInfo = new OmKeyInfo.Builder() - .setKeyName("file2") - .setVolumeName("vol") - .setBucketName("bucket1") - .setObjectID(2L) - .setParentObjectID(dirOneObjectId) - .build(); - - // Should return empty string when NSSummary has invalid parentId - String fullPath = ReconUtils.constructFullPath(keyInfo, mockSummaryManager); - Assertions.assertEquals("", fullPath, "Should return empty string when NSSummary has negative parentId"); - } - - @Test - public void testDataSizeUnderVolumeWithRatisReplication()throws IOException { - Response keyResponse = nsSummaryEndpoint.getDiskUsage(VOL_THREE_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(QuotaUtil.getReplicatedSize(replicaDUResponse.getSize(), RatisReplicationConfig.getInstance( - HddsProtos.ReplicationFactor.THREE)), replicaDUResponse.getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderBucketWithRatisReplication()throws IOException { - Response keyResponse = nsSummaryEndpoint.getDiskUsage(VOL_THREE_PATH + "/" + BUCKET_FIVE, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(QuotaUtil.getReplicatedSize(replicaDUResponse.getSize(), RatisReplicationConfig.getInstance( - HddsProtos.ReplicationFactor.THREE)), replicaDUResponse.getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderDirWithRatisReplication()throws IOException { - Response keyResponse = nsSummaryEndpoint.getDiskUsage(VOL_THREE_PATH + "/" + BUCKET_FIVE + "/" + DIR_SIX, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(QuotaUtil.getReplicatedSize(replicaDUResponse.getSize(), RatisReplicationConfig.getInstance( - HddsProtos.ReplicationFactor.THREE)), replicaDUResponse.getSizeWithReplica()); - } - - /** - * Write directories and keys info into OM DB. - * @throws Exception - */ - private void populateOMDB() throws Exception { - // write all directories - writeDirToOm(reconOMMetadataManager, DIR_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, DIR_ONE); - writeDirToOm(reconOMMetadataManager, DIR_TWO_OBJECT_ID, - DIR_ONE_OBJECT_ID, BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, DIR_TWO); - writeDirToOm(reconOMMetadataManager, DIR_THREE_OBJECT_ID, - DIR_ONE_OBJECT_ID, BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, DIR_THREE); - writeDirToOm(reconOMMetadataManager, DIR_FOUR_OBJECT_ID, - DIR_ONE_OBJECT_ID, BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, DIR_FOUR); - writeDirToOm(reconOMMetadataManager, DIR_FIVE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, DIR_FIVE); - - // write all keys - writeKeyToOm(reconOMMetadataManager, - KEY_ONE, - BUCKET_ONE, - VOL, - FILE_ONE, - KEY_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - KEY_ONE_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_TWO, - BUCKET_ONE, - VOL, - FILE_TWO, - KEY_TWO_OBJECT_ID, - DIR_TWO_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - KEY_TWO_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_THREE, - BUCKET_ONE, - VOL, - FILE_THREE, - KEY_THREE_OBJECT_ID, - DIR_THREE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - KEY_THREE_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_FOUR, - BUCKET_TWO, - VOL, - FILE_FOUR, - KEY_FOUR_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - VOL_OBJECT_ID, - KEY_FOUR_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_FIVE, - BUCKET_TWO, - VOL, - FILE_FIVE, - KEY_FIVE_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - VOL_OBJECT_ID, - KEY_FIVE_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_SIX, - BUCKET_ONE, - VOL, - FILE_SIX, - KEY_SIX_OBJECT_ID, - DIR_FOUR_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - KEY_SIX_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_EIGHT, - BUCKET_THREE, - VOL_TWO, - FILE_EIGHT, - KEY_EIGHT_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - KEY_EIGHT_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_NINE, - BUCKET_THREE, - VOL_TWO, - FILE_NINE, - KEY_NINE_OBJECT_ID, - DIR_FIVE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - KEY_NINE_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_TEN, - BUCKET_THREE, - VOL_TWO, - FILE_TEN, - KEY_TEN_OBJECT_ID, - DIR_FIVE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - KEY_TEN_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_ELEVEN, - BUCKET_FOUR, - VOL_TWO, - FILE_ELEVEN, - KEY_ELEVEN_OBJECT_ID, - BUCKET_FOUR_OBJECT_ID, - BUCKET_FOUR_OBJECT_ID, - VOL_TWO_OBJECT_ID, - KEY_ELEVEN_SIZE, - getBucketLayout()); - - } - - private void populateVolumeThree() throws IOException { - - writeDirToOm(reconOMMetadataManager, DIR_SIX_OBJECT_ID, - BUCKET_FIVE_OBJECT_ID, BUCKET_FIVE_OBJECT_ID, - VOL_THREE_OBJECT_ID, DIR_SIX); - - writeDirToOm(reconOMMetadataManager, DIR_SEVEN_OBJECT_ID, - DIR_SIX_OBJECT_ID, BUCKET_FIVE_OBJECT_ID, - VOL_THREE_OBJECT_ID, DIR_SEVEN); - - writeKeyToOm(reconOMMetadataManager, - KEY_TWELVE, - BUCKET_FIVE, - VOL_THREE, - FILE_TWELVE, - FILE_TWELVE_OBJECT_ID, - BUCKET_FIVE_OBJECT_ID, - BUCKET_FIVE_OBJECT_ID, - VOL_THREE_OBJECT_ID, - KEY_TWELVE_SIZE, - getBucketLayout(), RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), - Time.now(), true); - - writeKeyToOm(reconOMMetadataManager, - KEY_THIRTEEN, - BUCKET_FIVE, - VOL_THREE, - FILE_THIRTEEN, - FILE_THIRTEEN_OBJECT_ID, - DIR_SIX_OBJECT_ID, - BUCKET_FIVE_OBJECT_ID, - VOL_THREE_OBJECT_ID, - KEY_THIRTEEN_SIZE, - getBucketLayout(), RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), - Time.now(), true); - - writeKeyToOm(reconOMMetadataManager, - KEY_FOURTEEN, - BUCKET_FIVE, - VOL_THREE, - FILE_FOURTEEN, - FILE_FOURTEEN_OBJECT_ID, - DIR_SEVEN_OBJECT_ID, - BUCKET_FIVE_OBJECT_ID, - VOL_THREE_OBJECT_ID, - KEY_FOURTEEN_SIZE, - getBucketLayout(), RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), - Time.now(), true); - } - - /** - * Create a new OM Metadata manager instance with one user, one vol, and two - * buckets. - * @throws IOException ioEx - */ - private static OMMetadataManager initializeNewOmMetadataManager( - File omDbDir) - throws IOException { - OzoneConfiguration omConfiguration = new OzoneConfiguration(); - omConfiguration.set(OZONE_OM_DB_DIRS, - omDbDir.getAbsolutePath()); - OMMetadataManager omMetadataManager = new OmMetadataManagerImpl( - omConfiguration, null); - - String volumeKey = omMetadataManager.getVolumeKey(VOL); - OmVolumeArgs args = - OmVolumeArgs.newBuilder() - .setObjectID(VOL_OBJECT_ID) - .setVolume(VOL) - .setAdminName(TEST_USER) - .setOwnerName(TEST_USER) - .setQuotaInBytes(VOL_QUOTA) - .build(); - - String volume2Key = omMetadataManager.getVolumeKey(VOL_TWO); - OmVolumeArgs args2 = - OmVolumeArgs.newBuilder() - .setObjectID(VOL_TWO_OBJECT_ID) - .setVolume(VOL_TWO) - .setAdminName(TEST_USER) - .setOwnerName(TEST_USER) - .setQuotaInBytes(VOL_TWO_QUOTA) - .build(); - - String volume3Key = omMetadataManager.getVolumeKey(VOL_THREE); - OmVolumeArgs args3 = - OmVolumeArgs.newBuilder() - .setObjectID(VOL_THREE_OBJECT_ID) - .setVolume(VOL_THREE) - .setAdminName(TEST_USER) - .setOwnerName(TEST_USER) - .setQuotaInBytes(VOL_THREE_QUOTA) - .build(); - - omMetadataManager.getVolumeTable().put(volumeKey, args); - omMetadataManager.getVolumeTable().put(volume2Key, args2); - omMetadataManager.getVolumeTable().put(volume3Key, args3); - - OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() - .setVolumeName(VOL) - .setBucketName(BUCKET_ONE) - .setObjectID(BUCKET_ONE_OBJECT_ID) - .setQuotaInBytes(BUCKET_ONE_QUOTA) - .setBucketLayout(getBucketLayout()) - .build(); - - OmBucketInfo bucketInfo2 = OmBucketInfo.newBuilder() - .setVolumeName(VOL) - .setBucketName(BUCKET_TWO) - .setObjectID(BUCKET_TWO_OBJECT_ID) - .setQuotaInBytes(BUCKET_TWO_QUOTA) - .setBucketLayout(getBucketLayout()) - .build(); - - OmBucketInfo bucketInfo3 = OmBucketInfo.newBuilder() - .setVolumeName(VOL_TWO) - .setBucketName(BUCKET_THREE) - .setObjectID(BUCKET_THREE_OBJECT_ID) - .setQuotaInBytes(BUCKET_THREE_QUOTA) - .setBucketLayout(getBucketLayout()) - .build(); - - OmBucketInfo bucketInfo4 = OmBucketInfo.newBuilder() - .setVolumeName(VOL_TWO) - .setBucketName(BUCKET_FOUR) - .setObjectID(BUCKET_FOUR_OBJECT_ID) - .setQuotaInBytes(BUCKET_FOUR_QUOTA) - .setBucketLayout(getBucketLayout()) - .build(); - - OmBucketInfo bucketInfo5 = OmBucketInfo.newBuilder() - .setVolumeName(VOL_THREE) - .setBucketName(BUCKET_FIVE) - .setObjectID(BUCKET_FIVE_OBJECT_ID) - .setQuotaInBytes(BUCKET_FIVE_QUOTA) - .setBucketLayout(getBucketLayout()) - .build(); - - String bucketKey = omMetadataManager.getBucketKey( - bucketInfo.getVolumeName(), bucketInfo.getBucketName()); - String bucketKey2 = omMetadataManager.getBucketKey( - bucketInfo2.getVolumeName(), bucketInfo2.getBucketName()); - String bucketKey3 = omMetadataManager.getBucketKey( - bucketInfo3.getVolumeName(), bucketInfo3.getBucketName()); - String bucketKey4 = omMetadataManager.getBucketKey( - bucketInfo4.getVolumeName(), bucketInfo4.getBucketName()); - String bucketKey5 = omMetadataManager.getBucketKey( - bucketInfo5.getVolumeName(), bucketInfo5.getBucketName()); - - omMetadataManager.getBucketTable().put(bucketKey, bucketInfo); - omMetadataManager.getBucketTable().put(bucketKey2, bucketInfo2); - omMetadataManager.getBucketTable().put(bucketKey3, bucketInfo3); - omMetadataManager.getBucketTable().put(bucketKey4, bucketInfo4); - omMetadataManager.getBucketTable().put(bucketKey5, bucketInfo5); - - return omMetadataManager; - } - - private void setUpMultiBlockKey() throws IOException { - OmKeyLocationInfoGroup locationInfoGroup = - getLocationInfoGroup1(); - - // add the multi-block key to Recon's OM - writeKeyToOm(reconOMMetadataManager, - MULTI_BLOCK_KEY, - BUCKET_ONE, - VOL, - MULTI_BLOCK_FILE, - MULTI_BLOCK_KEY_OBJECT_ID, - DIR_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup), - getBucketLayout(), - KEY_SEVEN_SIZE); - } - - private OmKeyLocationInfoGroup getLocationInfoGroup1() { - List<OmKeyLocationInfo> locationInfoList = new ArrayList<>(); - BlockID block1 = new BlockID(CONTAINER_ONE_ID, 0L); - BlockID block2 = new BlockID(CONTAINER_TWO_ID, 0L); - BlockID block3 = new BlockID(CONTAINER_THREE_ID, 0L); - - OmKeyLocationInfo location1 = new OmKeyLocationInfo.Builder() - .setBlockID(block1) - .setLength(BLOCK_ONE_LENGTH) - .build(); - OmKeyLocationInfo location2 = new OmKeyLocationInfo.Builder() - .setBlockID(block2) - .setLength(BLOCK_TWO_LENGTH) - .build(); - OmKeyLocationInfo location3 = new OmKeyLocationInfo.Builder() - .setBlockID(block3) - .setLength(BLOCK_THREE_LENGTH) - .build(); - locationInfoList.add(location1); - locationInfoList.add(location2); - locationInfoList.add(location3); - - return new OmKeyLocationInfoGroup(0L, locationInfoList); - } - - /** - * Testing the following case. - * vol - * / \ - * bucket1 bucket2 - * / \ / \ - * file1 dir1 file4 file5 - * / \ \ \ - * dir2 dir3 dir4 file7 - * / \ \ - * file2 file3 file6 - * ---------------------------------------- - * vol2 - * / \ - * bucket3 bucket4 - * / \ / - * file8 dir5 file11 - * / \ - * file9 file10 - * Write these keys to OM and - * replicate them. - */ - private OmKeyLocationInfoGroup getLocationInfoGroup2() { - List<OmKeyLocationInfo> locationInfoList = new ArrayList<>(); - BlockID block4 = new BlockID(CONTAINER_FOUR_ID, 0L); - BlockID block5 = new BlockID(CONTAINER_FIVE_ID, 0L); - BlockID block6 = new BlockID(CONTAINER_SIX_ID, 0L); - - OmKeyLocationInfo location4 = new OmKeyLocationInfo.Builder() - .setBlockID(block4) - .setLength(BLOCK_FOUR_LENGTH) - .build(); - OmKeyLocationInfo location5 = new OmKeyLocationInfo.Builder() - .setBlockID(block5) - .setLength(BLOCK_FIVE_LENGTH) - .build(); - OmKeyLocationInfo location6 = new OmKeyLocationInfo.Builder() - .setBlockID(block6) - .setLength(BLOCK_SIX_LENGTH) - .build(); - locationInfoList.add(location4); - locationInfoList.add(location5); - locationInfoList.add(location6); - return new OmKeyLocationInfoGroup(0L, locationInfoList); - - } - - @SuppressWarnings("checkstyle:MethodLength") - private void setUpMultiBlockReplicatedKeys() throws IOException { - OmKeyLocationInfoGroup locationInfoGroup1 = - getLocationInfoGroup1(); - OmKeyLocationInfoGroup locationInfoGroup2 = - getLocationInfoGroup2(); - - //vol/bucket1/file1 - writeKeyToOm(reconOMMetadataManager, - KEY_ONE, - BUCKET_ONE, - VOL, - FILE_ONE, - KEY_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getBucketLayout(), - KEY_ONE_SIZE); - - //vol/bucket1/dir1/dir2/file2 - writeKeyToOm(reconOMMetadataManager, - KEY_TWO, - BUCKET_ONE, - VOL, - FILE_TWO, - KEY_TWO_OBJECT_ID, - DIR_TWO_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getBucketLayout(), - KEY_TWO_SIZE); - - //vol/bucket1/dir1/dir3/file3 - writeKeyToOm(reconOMMetadataManager, - KEY_THREE, - BUCKET_ONE, - VOL, - FILE_THREE, - KEY_THREE_OBJECT_ID, - DIR_THREE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getBucketLayout(), - KEY_THREE_SIZE); - - //vol/bucket2/file4 - writeKeyToOm(reconOMMetadataManager, - KEY_FOUR, - BUCKET_TWO, - VOL, - FILE_FOUR, - KEY_FOUR_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getBucketLayout(), - KEY_FOUR_SIZE); - - //vol/bucket2/file5 - writeKeyToOm(reconOMMetadataManager, - KEY_FIVE, - BUCKET_TWO, - VOL, - FILE_FIVE, - KEY_FIVE_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getBucketLayout(), - KEY_FIVE_SIZE); - - //vol/bucket1/dir1/dir4/file6 - writeKeyToOm(reconOMMetadataManager, - KEY_SIX, - BUCKET_ONE, - VOL, - FILE_SIX, - KEY_SIX_OBJECT_ID, - DIR_FOUR_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getBucketLayout(), - KEY_SIX_SIZE); - - //vol/bucket1/dir1/file7 - writeKeyToOm(reconOMMetadataManager, - KEY_SEVEN, - BUCKET_ONE, - VOL, - FILE_SEVEN, - KEY_SEVEN_OBJECT_ID, - DIR_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getBucketLayout(), - KEY_SEVEN_SIZE); - - //vol2/bucket3/file8 - writeKeyToOm(reconOMMetadataManager, - KEY_EIGHT, - BUCKET_THREE, - VOL_TWO, - FILE_EIGHT, - KEY_EIGHT_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getBucketLayout(), - KEY_EIGHT_SIZE); - - //vol2/bucket3/dir5/file9 - writeKeyToOm(reconOMMetadataManager, - KEY_NINE, - BUCKET_THREE, - VOL_TWO, - FILE_NINE, - KEY_NINE_OBJECT_ID, - DIR_FIVE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getBucketLayout(), - KEY_NINE_SIZE); - - //vol2/bucket3/dir5/file10 - writeKeyToOm(reconOMMetadataManager, - KEY_TEN, - BUCKET_THREE, - VOL_TWO, - FILE_TEN, - KEY_TEN_OBJECT_ID, - DIR_FIVE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getBucketLayout(), - KEY_TEN_SIZE); - - //vol2/bucket4/file11 - writeKeyToOm(reconOMMetadataManager, - KEY_ELEVEN, - BUCKET_FOUR, - VOL_TWO, - FILE_ELEVEN, - KEY_ELEVEN_OBJECT_ID, - BUCKET_FOUR_OBJECT_ID, - BUCKET_FOUR_OBJECT_ID, - VOL_TWO_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getBucketLayout(), - KEY_ELEVEN_SIZE); - } - - /** - * Generate a set of mock container replica with a size of - * replication factor for container. - * @param replicationFactor number of replica - * @param containerID the container replicated based upon - * @return a set of container replica for testing - */ - private static Set<ContainerReplica> generateMockContainerReplicas( - int replicationFactor, ContainerID containerID) { - Set<ContainerReplica> result = new HashSet<>(); - for (int i = 0; i < replicationFactor; ++i) { - DatanodeDetails randomDatanode = randomDatanodeDetails(); - ContainerReplica replica = new ContainerReplica.ContainerReplicaBuilder() - .setContainerID(containerID) - .setContainerState(State.OPEN) - .setDatanodeDetails(randomDatanode) - .build(); - result.add(replica); - } - return result; - } - - private static ReconStorageContainerManagerFacade getMockReconSCM() - throws ContainerNotFoundException { - ReconStorageContainerManagerFacade reconSCM = - mock(ReconStorageContainerManagerFacade.class); - ContainerManager containerManager = mock(ContainerManager.class); - - // Container 1 is 3-way replicated - ContainerID containerID1 = ContainerID.valueOf(CONTAINER_ONE_ID); - Set<ContainerReplica> containerReplicas1 = generateMockContainerReplicas( - CONTAINER_ONE_REPLICA_COUNT, containerID1); - when(containerManager.getContainerReplicas(containerID1)) - .thenReturn(containerReplicas1); - - // Container 2 is under replicated with 2 replica - ContainerID containerID2 = ContainerID.valueOf(CONTAINER_TWO_ID); - Set<ContainerReplica> containerReplicas2 = generateMockContainerReplicas( - CONTAINER_TWO_REPLICA_COUNT, containerID2); - when(containerManager.getContainerReplicas(containerID2)) - .thenReturn(containerReplicas2); - - // Container 3 is over replicated with 4 replica - ContainerID containerID3 = ContainerID.valueOf(CONTAINER_THREE_ID); - Set<ContainerReplica> containerReplicas3 = generateMockContainerReplicas( - CONTAINER_THREE_REPLICA_COUNT, containerID3); - when(containerManager.getContainerReplicas(containerID3)) - .thenReturn(containerReplicas3); - - // Container 4 is replicated with 5 replica - ContainerID containerID4 = ContainerID.valueOf(CONTAINER_FOUR_ID); - Set<ContainerReplica> containerReplicas4 = generateMockContainerReplicas( - CONTAINER_FOUR_REPLICA_COUNT, containerID4); - when(containerManager.getContainerReplicas(containerID4)) - .thenReturn(containerReplicas4); - - // Container 5 is replicated with 2 replica - ContainerID containerID5 = ContainerID.valueOf(CONTAINER_FIVE_ID); - Set<ContainerReplica> containerReplicas5 = generateMockContainerReplicas( - CONTAINER_FIVE_REPLICA_COUNT, containerID5); - when(containerManager.getContainerReplicas(containerID5)) - .thenReturn(containerReplicas5); - - // Container 6 is replicated with 3 replica - ContainerID containerID6 = ContainerID.valueOf(CONTAINER_SIX_ID); - Set<ContainerReplica> containerReplicas6 = generateMockContainerReplicas( - CONTAINER_SIX_REPLICA_COUNT, containerID6); - when(containerManager.getContainerReplicas(containerID6)) - .thenReturn(containerReplicas6); - - when(reconSCM.getContainerManager()).thenReturn(containerManager); - ReconNodeManager mockReconNodeManager = mock(ReconNodeManager.class); - when(mockReconNodeManager.getStats()).thenReturn(getMockSCMRootStat()); - when(reconSCM.getScmNodeManager()).thenReturn(mockReconNodeManager); - return reconSCM; - } - - private static BucketLayout getBucketLayout() { - return BucketLayout.FILE_SYSTEM_OPTIMIZED; - } - - private static SCMNodeStat getMockSCMRootStat() { - return new SCMNodeStat(ROOT_QUOTA, ROOT_DATA_SIZE, - ROOT_QUOTA - ROOT_DATA_SIZE, 0, ROOT_QUOTA - ROOT_DATA_SIZE - 1, 0); - } -}
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java deleted file mode 100644 index 7ac14a9..0000000 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java +++ /dev/null
@@ -1,1491 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.recon.api; - -import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; -import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; -import static org.apache.hadoop.ozone.om.helpers.QuotaUtil.getReplicatedSize; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProvider; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.setConfiguration; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDirToOm; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import javax.ws.rs.core.Response; -import org.apache.hadoop.hdds.client.BlockID; -import org.apache.hadoop.hdds.client.RatisReplicationConfig; -import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State; -import org.apache.hadoop.hdds.scm.container.ContainerID; -import org.apache.hadoop.hdds.scm.container.ContainerManager; -import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; -import org.apache.hadoop.hdds.scm.container.ContainerReplica; -import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat; -import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; -import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; -import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; -import org.apache.hadoop.ozone.om.helpers.QuotaUtil; -import org.apache.hadoop.ozone.recon.ReconConstants; -import org.apache.hadoop.ozone.recon.ReconTestInjector; -import org.apache.hadoop.ozone.recon.ReconUtils; -import org.apache.hadoop.ozone.recon.api.handlers.BucketHandler; -import org.apache.hadoop.ozone.recon.api.handlers.EntityHandler; -import org.apache.hadoop.ozone.recon.api.types.DUResponse; -import org.apache.hadoop.ozone.recon.api.types.FileSizeDistributionResponse; -import org.apache.hadoop.ozone.recon.api.types.QuotaUsageResponse; -import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; -import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; -import org.apache.hadoop.ozone.recon.scm.ReconNodeManager; -import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; -import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; -import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; -import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; -import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; -import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithLegacy; -import org.apache.hadoop.util.Time; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -/** - * Test for NSSummary REST APIs with Legacy. - * We tested on a mini file system with the following setting: - * vol - * / \ - * bucket1 bucket2 - * / \ / \ - * file1 dir1 file4 file5 - * / \ \ - * dir2 dir3 dir4 - * / \ \ - * file2 file3 file6 - * ---------------------------------------- - * vol2 - * / \ - * bucket3 bucket4 - * / \ / - * file8 dir5 file11 - * / \ - * file9 file10 - * ---------------------------------------- - * vol3 - * | - * bucket5 - * / \ - * file12 dir6 - * / \ - * file13 dir7 - * / - * file14 - * This is a test for the Rest APIs only. We have tested NSSummaryTask before, - * so there is no need to test process() on DB's updates - */ -public class TestNSSummaryEndpointWithLegacy extends NSSummaryTests { - @TempDir - private Path temporaryFolder; - - private ReconNamespaceSummaryManager reconNamespaceSummaryManager; - private ReconOMMetadataManager reconOMMetadataManager; - private NSSummaryEndpoint nsSummaryEndpoint; - - private static final String TEST_PATH_UTILITY = - "/vol1/buck1/a/b/c/d/e/file1.txt"; - private static final String PARENT_DIR = "vol1/buck1/a/b/c/d/e"; - private static final String[] TEST_NAMES = - new String[]{"vol1", "buck1", "a", "b", "c", "d", "e", "file1.txt"}; - private static final String TEST_KEY_NAMES = "a/b/c/d/e/file1.txt"; - - // Object names - private static final String VOL = "vol"; - private static final String VOL_TWO = "vol2"; - private static final String VOL_THREE = "vol3"; - private static final String BUCKET_ONE = "bucket1"; - private static final String BUCKET_TWO = "bucket2"; - private static final String BUCKET_THREE = "bucket3"; - private static final String BUCKET_FOUR = "bucket4"; - private static final String BUCKET_FIVE = "bucket5"; - private static final String KEY_ONE = "file1"; - private static final String KEY_TWO = "dir1/dir2/file2"; - private static final String KEY_THREE = "dir1/dir3/file3"; - private static final String KEY_FOUR = "file4"; - private static final String KEY_FIVE = "file5"; - private static final String KEY_SIX = "dir1/dir4/file6"; - private static final String KEY_SEVEN = "dir1/file7"; - private static final String KEY_EIGHT = "file8"; - private static final String KEY_NINE = "dir5/file9"; - private static final String KEY_TEN = "dir5/file10"; - private static final String KEY_ELEVEN = "file11"; - private static final String MULTI_BLOCK_KEY = "dir1/file7"; - private static final String MULTI_BLOCK_FILE = "file7"; - private static final String KEY_TWELVE = "file12"; - private static final String KEY_THIRTEEN = "dir6/file13"; - private static final String KEY_FOURTEEN = "dir6/dir7/file14"; - - private static final String FILE_ONE = "file1"; - private static final String FILE_TWO = "file2"; - private static final String FILE_THREE = "file3"; - private static final String FILE_FOUR = "file4"; - private static final String FILE_FIVE = "file5"; - private static final String FILE_SIX = "file6"; - private static final String FILE_SEVEN = "file7"; - private static final String FILE_EIGHT = "file8"; - private static final String FILE_NINE = "file9"; - private static final String FILE_TEN = "file10"; - private static final String FILE_ELEVEN = "file11"; - private static final String FILE_TWELVE = "file12"; - private static final String FILE_THIRTEEN = "file13"; - private static final String FILE_FOURTEEN = "file14"; - - private static final String DIR_ONE = "dir1"; - private static final String DIR_TWO = "dir2"; - private static final String DIR_THREE = "dir3"; - private static final String DIR_FOUR = "dir4"; - private static final String DIR_FIVE = "dir5"; - private static final String DIR_SIX = "dir6"; - private static final String DIR_SEVEN = "dir7"; - // objects IDs - private static final long PARENT_OBJECT_ID_ZERO = 0L; - private static final long VOL_OBJECT_ID = 0L; - private static final long BUCKET_ONE_OBJECT_ID = 1L; - private static final long BUCKET_TWO_OBJECT_ID = 2L; - private static final long KEY_ONE_OBJECT_ID = 3L; - private static final long DIR_ONE_OBJECT_ID = 4L; - private static final long KEY_TWO_OBJECT_ID = 5L; - private static final long KEY_FOUR_OBJECT_ID = 6L; - private static final long DIR_TWO_OBJECT_ID = 7L; - private static final long KEY_THREE_OBJECT_ID = 8L; - private static final long KEY_FIVE_OBJECT_ID = 9L; - private static final long KEY_SIX_OBJECT_ID = 10L; - private static final long DIR_THREE_OBJECT_ID = 11L; - private static final long DIR_FOUR_OBJECT_ID = 12L; - private static final long MULTI_BLOCK_KEY_OBJECT_ID = 13L; - private static final long KEY_SEVEN_OBJECT_ID = 13L; - private static final long VOL_TWO_OBJECT_ID = 14L; - private static final long BUCKET_THREE_OBJECT_ID = 15L; - private static final long BUCKET_FOUR_OBJECT_ID = 16L; - private static final long KEY_EIGHT_OBJECT_ID = 17L; - private static final long DIR_FIVE_OBJECT_ID = 18L; - private static final long KEY_NINE_OBJECT_ID = 19L; - private static final long KEY_TEN_OBJECT_ID = 20L; - private static final long KEY_ELEVEN_OBJECT_ID = 21L; - private static final long VOL_THREE_OBJECT_ID = 22L; - private static final long DIR_SIX_OBJECT_ID = 23L; - private static final long DIR_SEVEN_OBJECT_ID = 24L; - private static final long FILE_TWELVE_OBJECT_ID = 25L; - private static final long FILE_THIRTEEN_OBJECT_ID = 26L; - private static final long FILE_FOURTEEN_OBJECT_ID = 27L; - private static final long BUCKET_FIVE_OBJECT_ID = 28L; - - // container IDs - private static final long CONTAINER_ONE_ID = 1L; - private static final long CONTAINER_TWO_ID = 2L; - private static final long CONTAINER_THREE_ID = 3L; - private static final long CONTAINER_FOUR_ID = 4L; - private static final long CONTAINER_FIVE_ID = 5L; - private static final long CONTAINER_SIX_ID = 6L; - - // replication factors - private static final int CONTAINER_ONE_REPLICA_COUNT = 3; - private static final int CONTAINER_TWO_REPLICA_COUNT = 2; - private static final int CONTAINER_THREE_REPLICA_COUNT = 4; - private static final int CONTAINER_FOUR_REPLICA_COUNT = 5; - private static final int CONTAINER_FIVE_REPLICA_COUNT = 2; - private static final int CONTAINER_SIX_REPLICA_COUNT = 3; - - // block lengths - private static final long BLOCK_ONE_LENGTH = 1000L; - private static final long BLOCK_TWO_LENGTH = 2000L; - private static final long BLOCK_THREE_LENGTH = 3000L; - private static final long BLOCK_FOUR_LENGTH = 4000L; - private static final long BLOCK_FIVE_LENGTH = 5000L; - private static final long BLOCK_SIX_LENGTH = 6000L; - - // data size in bytes - private static final long KEY_ONE_SIZE = 500L; // bin 0 - private static final long KEY_TWO_SIZE = OzoneConsts.KB + 1; // bin 1 - private static final long KEY_THREE_SIZE = 4 * OzoneConsts.KB + 1; // bin 3 - private static final long KEY_FOUR_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 - private static final long KEY_FIVE_SIZE = 100L; // bin 0 - private static final long KEY_SIX_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 - private static final long KEY_SEVEN_SIZE = 4 * OzoneConsts.KB + 1; - private static final long KEY_EIGHT_SIZE = OzoneConsts.KB + 1; // bin 1 - private static final long KEY_NINE_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 - private static final long KEY_TEN_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 - private static final long KEY_ELEVEN_SIZE = OzoneConsts.KB + 1; // bin 1 - private static final long KEY_TWELVE_SIZE = OzoneConsts.KB; - private static final long KEY_THIRTEEN_SIZE = OzoneConsts.KB; - private static final long KEY_FOURTEEN_SIZE = OzoneConsts.KB; - - private static final long FILE1_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_ONE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE2_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_TWO_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE3_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_THREE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE4_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_FOUR_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE5_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_FIVE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE6_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_SIX_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE7_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_SEVEN_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE8_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_EIGHT_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE9_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_NINE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE10_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_TEN_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE11_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_ELEVEN_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE12_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_TWELVE_SIZE, RatisReplicationConfig.getInstance(THREE)); - private static final long FILE13_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_THIRTEEN_SIZE, RatisReplicationConfig.getInstance(THREE)); - private static final long FILE14_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_FOURTEEN_SIZE, RatisReplicationConfig.getInstance(THREE)); - - private static final long MULTI_BLOCK_KEY_SIZE_WITH_REPLICA - = FILE7_SIZE_WITH_REPLICA; - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_ROOT - = FILE1_SIZE_WITH_REPLICA - + FILE2_SIZE_WITH_REPLICA - + FILE3_SIZE_WITH_REPLICA - + FILE4_SIZE_WITH_REPLICA - + FILE5_SIZE_WITH_REPLICA - + FILE6_SIZE_WITH_REPLICA - + FILE7_SIZE_WITH_REPLICA - + FILE8_SIZE_WITH_REPLICA - + FILE9_SIZE_WITH_REPLICA - + FILE10_SIZE_WITH_REPLICA - + FILE11_SIZE_WITH_REPLICA - + FILE12_SIZE_WITH_REPLICA - + FILE13_SIZE_WITH_REPLICA - + FILE14_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_VOL - = FILE1_SIZE_WITH_REPLICA - + FILE2_SIZE_WITH_REPLICA - + FILE3_SIZE_WITH_REPLICA - + FILE4_SIZE_WITH_REPLICA - + FILE5_SIZE_WITH_REPLICA - + FILE6_SIZE_WITH_REPLICA - + FILE7_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET1 - = FILE1_SIZE_WITH_REPLICA - + FILE2_SIZE_WITH_REPLICA - + FILE3_SIZE_WITH_REPLICA - + FILE6_SIZE_WITH_REPLICA - + FILE7_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR1 - = FILE2_SIZE_WITH_REPLICA - + FILE3_SIZE_WITH_REPLICA - + FILE6_SIZE_WITH_REPLICA - + FILE7_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR2 - = FILE2_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_KEY - = FILE4_SIZE_WITH_REPLICA; - - // quota in bytes - private static final long ROOT_QUOTA = 2 * (2 * OzoneConsts.MB); - private static final long VOL_QUOTA = 2 * OzoneConsts.MB; - private static final long VOL_TWO_QUOTA = 2 * OzoneConsts.MB; - private static final long VOL_THREE_QUOTA = 2 * OzoneConsts.MB; - private static final long BUCKET_ONE_QUOTA = OzoneConsts.MB; - private static final long BUCKET_TWO_QUOTA = OzoneConsts.MB; - private static final long BUCKET_THREE_QUOTA = OzoneConsts.MB; - private static final long BUCKET_FOUR_QUOTA = OzoneConsts.MB; - private static final long BUCKET_FIVE_QUOTA = OzoneConsts.MB; - - // mock client's path requests - private static final String TEST_USER = "TestUser"; - private static final String ROOT_PATH = "/"; - private static final String VOL_PATH = "/vol"; - private static final String VOL_TWO_PATH = "/vol2"; - private static final String BUCKET_ONE_PATH = "/vol/bucket1"; - private static final String BUCKET_TWO_PATH = "/vol/bucket2"; - private static final String DIR_ONE_PATH = "/vol/bucket1/dir1"; - private static final String DIR_TWO_PATH = "/vol/bucket1/dir1/dir2"; - private static final String DIR_THREE_PATH = "/vol/bucket1/dir1/dir3"; - private static final String DIR_FOUR_PATH = "/vol/bucket1/dir1/dir4"; - private static final String KEY_PATH = "/vol/bucket2/file4"; - private static final String MULTI_BLOCK_KEY_PATH = "/vol/bucket1/dir1/file7"; - private static final String INVALID_PATH = "/vol/path/not/found"; - private static final String VOL_THREE_PATH = "/vol3"; - - // some expected answers - private static final long ROOT_DATA_SIZE = KEY_ONE_SIZE + KEY_TWO_SIZE + - KEY_THREE_SIZE + KEY_FOUR_SIZE + KEY_FIVE_SIZE + KEY_SIX_SIZE + KEY_SEVEN_SIZE + - KEY_EIGHT_SIZE + KEY_NINE_SIZE + KEY_TEN_SIZE + KEY_ELEVEN_SIZE + - FILE12_SIZE_WITH_REPLICA + FILE13_SIZE_WITH_REPLICA + FILE14_SIZE_WITH_REPLICA; - private static final long VOL_DATA_SIZE = KEY_ONE_SIZE + KEY_TWO_SIZE + - KEY_THREE_SIZE + KEY_FOUR_SIZE + KEY_FIVE_SIZE + KEY_SIX_SIZE + KEY_SEVEN_SIZE; - - private static final long VOL_TWO_DATA_SIZE = - KEY_EIGHT_SIZE + KEY_NINE_SIZE + KEY_TEN_SIZE + KEY_ELEVEN_SIZE; - - private static final long BUCKET_ONE_DATA_SIZE = KEY_ONE_SIZE + KEY_TWO_SIZE + - KEY_THREE_SIZE + KEY_SIX_SIZE + KEY_SEVEN_SIZE; - - private static final long BUCKET_TWO_DATA_SIZE = - KEY_FOUR_SIZE + KEY_FIVE_SIZE; - - private static final long DIR_ONE_DATA_SIZE = KEY_TWO_SIZE + - KEY_THREE_SIZE + KEY_SIX_SIZE + KEY_SEVEN_SIZE; - - @BeforeEach - public void setUp() throws Exception { - OzoneConfiguration conf = new OzoneConfiguration(); - OMMetadataManager omMetadataManager = initializeNewOmMetadataManager( - Files.createDirectory(temporaryFolder.resolve( - "JunitOmDBDir")).toFile(), conf); - OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = - getMockOzoneManagerServiceProvider(); - reconOMMetadataManager = getTestReconOmMetadataManager(omMetadataManager, - Files.createDirectory(temporaryFolder.resolve( - "omMetadatDir")).toFile()); - - ReconTestInjector reconTestInjector = - new ReconTestInjector.Builder(temporaryFolder.toFile()) - .withReconOm(reconOMMetadataManager) - .withOmServiceProvider(ozoneManagerServiceProvider) - .withReconSqlDb() - .withContainerDB() - .addBinding(OzoneStorageContainerManager.class, - getMockReconSCM()) - .addBinding(StorageContainerServiceProvider.class, - mock(StorageContainerServiceProviderImpl.class)) - .addBinding(NSSummaryEndpoint.class) - .build(); - this.reconNamespaceSummaryManager = - reconTestInjector.getInstance(ReconNamespaceSummaryManager.class); - nsSummaryEndpoint = reconTestInjector.getInstance(NSSummaryEndpoint.class); - - // populate OM DB and reprocess into Recon RocksDB - populateOMDB(); - populateVolumeThree(); - setUpMultiBlockReplicatedKeys(); - NSSummaryTaskWithLegacy nsSummaryTaskWithLegacy = - new NSSummaryTaskWithLegacy(reconNamespaceSummaryManager, - reconOMMetadataManager, conf, 10); - nsSummaryTaskWithLegacy.reprocessWithLegacy(reconOMMetadataManager); - } - - @Test - public void testUtility() { - String[] names = EntityHandler.parseRequestPath(TEST_PATH_UTILITY); - assertArrayEquals(TEST_NAMES, names); - String keyName = BucketHandler.getKeyName(names); - assertEquals(TEST_KEY_NAMES, keyName); - String subpath = BucketHandler.buildSubpath(PARENT_DIR, "file1.txt"); - assertEquals(TEST_PATH_UTILITY, subpath); - } - - @Test - public void testGetBasicInfoRoot() throws Exception { - testNSSummaryBasicInfoRoot( - nsSummaryEndpoint, reconOMMetadataManager); - } - - @Test - public void testGetBasicInfoVol() throws Exception { - // Test volume basics - testNSSummaryBasicInfoVolume(nsSummaryEndpoint); - } - - @Test - public void testGetBasicInfoBucketOne() throws Exception { - // Test bucket 1's basics - testNSSummaryBasicInfoBucketOne( - BucketLayout.LEGACY, - nsSummaryEndpoint); - } - - @Test - public void testGetBasicInfoBucketTwo() throws Exception { - // Test bucket 2's basics - testNSSummaryBasicInfoBucketTwo( - BucketLayout.LEGACY, - nsSummaryEndpoint); - } - - @Test - public void testGetBasicInfoDir() throws Exception { - // Test intermediate directory basics - testNSSummaryBasicInfoDir(nsSummaryEndpoint); - } - - @Test - public void testGetBasicInfoNoPath() throws Exception { - // Test invalid path - testNSSummaryBasicInfoNoPath(nsSummaryEndpoint); - } - - @Test - public void testGetBasicInfoKey() throws Exception { - // Test key - testNSSummaryBasicInfoKey(nsSummaryEndpoint); - } - - @Test - public void testDiskUsageRoot() throws Exception { - // root level DU - Response rootResponse = nsSummaryEndpoint.getDiskUsage(ROOT_PATH, - false, false, false); - DUResponse duRootRes = (DUResponse) rootResponse.getEntity(); - assertEquals(3, duRootRes.getCount()); - List<DUResponse.DiskUsage> duRootData = duRootRes.getDuData(); - // sort based on subpath - Collections.sort(duRootData, - Comparator.comparing(DUResponse.DiskUsage::getSubpath)); - DUResponse.DiskUsage duVol1 = duRootData.get(0); - DUResponse.DiskUsage duVol2 = duRootData.get(1); - assertEquals(VOL_PATH, duVol1.getSubpath()); - assertEquals(VOL_TWO_PATH, duVol2.getSubpath()); - assertEquals(VOL_DATA_SIZE, duVol1.getSize()); - assertEquals(VOL_TWO_DATA_SIZE, duVol2.getSize()); - } - - @Test - public void testDiskUsageVolume() throws Exception { - // volume level DU - Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, - false, false, false); - DUResponse duVolRes = (DUResponse) volResponse.getEntity(); - assertEquals(2, duVolRes.getCount()); - List<DUResponse.DiskUsage> duData = duVolRes.getDuData(); - // sort based on subpath - Collections.sort(duData, - Comparator.comparing(DUResponse.DiskUsage::getSubpath)); - DUResponse.DiskUsage duBucket1 = duData.get(0); - DUResponse.DiskUsage duBucket2 = duData.get(1); - assertEquals(BUCKET_ONE_PATH, duBucket1.getSubpath()); - assertEquals(BUCKET_TWO_PATH, duBucket2.getSubpath()); - assertEquals(BUCKET_ONE_DATA_SIZE, duBucket1.getSize()); - assertEquals(BUCKET_TWO_DATA_SIZE, duBucket2.getSize()); - } - - @Test - public void testDiskUsageBucket() throws Exception { - // bucket level DU - Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, - false, false, false); - DUResponse duBucketResponse = (DUResponse) bucketResponse.getEntity(); - assertEquals(1, duBucketResponse.getCount()); - DUResponse.DiskUsage duDir1 = duBucketResponse.getDuData().get(0); - assertEquals(DIR_ONE_PATH, duDir1.getSubpath()); - assertEquals(DIR_ONE_DATA_SIZE, duDir1.getSize()); - } - - @Test - public void testDiskUsageDir() throws Exception { - // dir level DU - Response dirResponse = nsSummaryEndpoint.getDiskUsage(DIR_ONE_PATH, - false, false, false); - DUResponse duDirReponse = (DUResponse) dirResponse.getEntity(); - assertEquals(3, duDirReponse.getCount()); - List<DUResponse.DiskUsage> duSubDir = duDirReponse.getDuData(); - Collections.sort(duSubDir, - Comparator.comparing(DUResponse.DiskUsage::getSubpath)); - DUResponse.DiskUsage duDir2 = duSubDir.get(0); - DUResponse.DiskUsage duDir3 = duSubDir.get(1); - DUResponse.DiskUsage duDir4 = duSubDir.get(2); - assertEquals(DIR_TWO_PATH, duDir2.getSubpath()); - assertEquals(KEY_TWO_SIZE, duDir2.getSize()); - - assertEquals(DIR_THREE_PATH, duDir3.getSubpath()); - assertEquals(KEY_THREE_SIZE, duDir3.getSize()); - - assertEquals(DIR_FOUR_PATH, duDir4.getSubpath()); - assertEquals(KEY_SIX_SIZE, duDir4.getSize()); - } - - @Test - public void testDiskUsageKey() throws Exception { - // key level DU - Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_PATH, - false, false, false); - DUResponse keyObj = (DUResponse) keyResponse.getEntity(); - assertEquals(0, keyObj.getCount()); - assertEquals(KEY_FOUR_SIZE, keyObj.getSize()); - } - - @Test - public void testDiskUsageUnknown() throws Exception { - // invalid path check - Response invalidResponse = nsSummaryEndpoint.getDiskUsage(INVALID_PATH, - false, false, false); - DUResponse invalidObj = (DUResponse) invalidResponse.getEntity(); - assertEquals(ResponseStatus.PATH_NOT_FOUND, - invalidObj.getStatus()); - } - - @Test - public void testDiskUsageWithReplication() throws Exception { - setUpMultiBlockKey(); - Response keyResponse = nsSummaryEndpoint.getDiskUsage(MULTI_BLOCK_KEY_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_KEY_SIZE_WITH_REPLICA, - replicaDUResponse.getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderRootWithReplication() throws IOException { - // withReplica is true - Response rootResponse = nsSummaryEndpoint.getDiskUsage(ROOT_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) rootResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_ROOT, - replicaDUResponse.getSizeWithReplica()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_VOL, - replicaDUResponse.getDuData().get(0).getSizeWithReplica()); - - } - - @Test - public void testDataSizeUnderVolWithReplication() throws IOException { - Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) volResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_VOL, - replicaDUResponse.getSizeWithReplica()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET1, - replicaDUResponse.getDuData().get(0).getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderBucketWithReplication() throws IOException { - Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) bucketResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET1, - replicaDUResponse.getSizeWithReplica()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR1, - replicaDUResponse.getDuData().get(0).getSizeWithReplica()); - } - - /** - * When calculating DU under dir1 - * there are 3 keys, file2, file3, file6. - * There is one direct key, file7. - * @throws IOException - */ - @Test - public void testDataSizeUnderDirWithReplication() throws IOException { - Response dir1Response = nsSummaryEndpoint.getDiskUsage(DIR_ONE_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) dir1Response.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR1, - replicaDUResponse.getSizeWithReplica()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR2, - replicaDUResponse.getDuData().get(0).getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderKeyWithReplication() throws IOException { - Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_KEY, - replicaDUResponse.getSizeWithReplica()); - } - - @Test - public void testQuotaUsage() throws Exception { - // root level quota usage - Response rootResponse = nsSummaryEndpoint.getQuotaUsage(ROOT_PATH); - QuotaUsageResponse quRootRes = - (QuotaUsageResponse) rootResponse.getEntity(); - assertEquals(ROOT_QUOTA, quRootRes.getQuota()); - assertEquals(ROOT_DATA_SIZE, quRootRes.getQuotaUsed()); - - // volume level quota usage - Response volResponse = nsSummaryEndpoint.getQuotaUsage(VOL_PATH); - QuotaUsageResponse quVolRes = (QuotaUsageResponse) volResponse.getEntity(); - assertEquals(VOL_QUOTA, quVolRes.getQuota()); - assertEquals(VOL_DATA_SIZE, quVolRes.getQuotaUsed()); - - // bucket level quota usage - Response bucketRes = nsSummaryEndpoint.getQuotaUsage(BUCKET_ONE_PATH); - QuotaUsageResponse quBucketRes = (QuotaUsageResponse) bucketRes.getEntity(); - assertEquals(BUCKET_ONE_QUOTA, quBucketRes.getQuota()); - assertEquals(BUCKET_ONE_DATA_SIZE, quBucketRes.getQuotaUsed()); - - Response bucketRes2 = nsSummaryEndpoint.getQuotaUsage(BUCKET_TWO_PATH); - QuotaUsageResponse quBucketRes2 = - (QuotaUsageResponse) bucketRes2.getEntity(); - assertEquals(BUCKET_TWO_QUOTA, quBucketRes2.getQuota()); - assertEquals(BUCKET_TWO_DATA_SIZE, quBucketRes2.getQuotaUsed()); - - // other level not applicable - Response naResponse1 = nsSummaryEndpoint.getQuotaUsage(DIR_ONE_PATH); - QuotaUsageResponse quotaUsageResponse1 = - (QuotaUsageResponse) naResponse1.getEntity(); - assertEquals(ResponseStatus.TYPE_NOT_APPLICABLE, - quotaUsageResponse1.getResponseCode()); - - Response naResponse2 = nsSummaryEndpoint.getQuotaUsage(KEY_PATH); - QuotaUsageResponse quotaUsageResponse2 = - (QuotaUsageResponse) naResponse2.getEntity(); - assertEquals(ResponseStatus.TYPE_NOT_APPLICABLE, - quotaUsageResponse2.getResponseCode()); - - // invalid path request - Response invalidRes = nsSummaryEndpoint.getQuotaUsage(INVALID_PATH); - QuotaUsageResponse invalidResObj = - (QuotaUsageResponse) invalidRes.getEntity(); - assertEquals(ResponseStatus.PATH_NOT_FOUND, - invalidResObj.getResponseCode()); - } - - @Test - public void testFileSizeDist() throws Exception { - checkFileSizeDist(ROOT_PATH, 5, 3, 4, 2); - checkFileSizeDist(VOL_PATH, 2, 1, 2, 2); - checkFileSizeDist(BUCKET_ONE_PATH, 1, 1, 1, 2); - checkFileSizeDist(DIR_ONE_PATH, 0, 1, 1, 2); - } - - public void checkFileSizeDist(String path, int bin0, - int bin1, int bin2, int bin3) throws Exception { - Response res = nsSummaryEndpoint.getFileSizeDistribution(path); - FileSizeDistributionResponse fileSizeDistResObj = - (FileSizeDistributionResponse) res.getEntity(); - int[] fileSizeDist = fileSizeDistResObj.getFileSizeDist(); - assertEquals(bin0, fileSizeDist[0]); - assertEquals(bin1, fileSizeDist[1]); - assertEquals(bin2, fileSizeDist[2]); - assertEquals(bin3, fileSizeDist[3]); - for (int i = 4; i < ReconConstants.NUM_OF_FILE_SIZE_BINS; ++i) { - assertEquals(0, fileSizeDist[i]); - } - } - - @Test - public void testConstructFullPath() throws IOException { - // For Key Tables the parent object ID is not set hence it - // will by default be set as -1 when the NSSummary object is created - OmKeyInfo keyInfo = new OmKeyInfo.Builder() - .setKeyName("dir1/dir2/file2") - .setVolumeName(VOL) - .setBucketName(BUCKET_ONE) - .setObjectID(KEY_TWO_OBJECT_ID) - .build(); - // Call constructFullPath and verify the result - String fullPath = ReconUtils.constructFullPath(keyInfo, - reconNamespaceSummaryManager); - String expectedPath = "vol/bucket1/dir1/dir2/file2"; - Assertions.assertEquals(expectedPath, fullPath); - - // Create key info for file 3 - keyInfo = new OmKeyInfo.Builder() - .setKeyName("dir1/dir2/") - .setVolumeName(VOL) - .setBucketName(BUCKET_ONE) - .setObjectID(DIR_TWO_OBJECT_ID) - .build(); - fullPath = ReconUtils.constructFullPath(keyInfo, - reconNamespaceSummaryManager); - expectedPath = "vol/bucket1/dir1/dir2/"; - Assertions.assertEquals(expectedPath, fullPath); - - // Create key info for file 6 - keyInfo = new OmKeyInfo.Builder() - .setKeyName("dir1/dir4/file6") - .setVolumeName(VOL) - .setBucketName(BUCKET_ONE) - .setObjectID(KEY_SIX_OBJECT_ID) - .build(); - fullPath = ReconUtils.constructFullPath(keyInfo, - reconNamespaceSummaryManager); - expectedPath = "vol/bucket1/dir1/dir4/file6"; - Assertions.assertEquals(expectedPath, fullPath); - } - - @Test - public void testDataSizeUnderVolumeWithRatisReplication()throws IOException { - Response keyResponse = nsSummaryEndpoint.getDiskUsage(VOL_THREE_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(QuotaUtil.getReplicatedSize(replicaDUResponse.getSize(), RatisReplicationConfig.getInstance( - HddsProtos.ReplicationFactor.THREE)), replicaDUResponse.getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderBucketWithRatisReplication()throws IOException { - Response keyResponse = nsSummaryEndpoint.getDiskUsage(VOL_THREE_PATH + "/" + BUCKET_FIVE, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(QuotaUtil.getReplicatedSize(replicaDUResponse.getSize(), RatisReplicationConfig.getInstance( - HddsProtos.ReplicationFactor.THREE)), replicaDUResponse.getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderDirWithRatisReplication()throws IOException { - Response keyResponse = nsSummaryEndpoint.getDiskUsage(VOL_THREE_PATH + "/" + BUCKET_FIVE + "/" + DIR_SIX, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(QuotaUtil.getReplicatedSize(replicaDUResponse.getSize(), RatisReplicationConfig.getInstance( - HddsProtos.ReplicationFactor.THREE)), replicaDUResponse.getSizeWithReplica()); - } - - /** - * Write directories and keys info into OM DB. - * @throws Exception - */ - @SuppressWarnings("checkstyle:MethodLength") - private void populateOMDB() throws Exception { - // write all directories - writeDirToOm(reconOMMetadataManager, - (DIR_ONE + OM_KEY_PREFIX), - BUCKET_ONE, - VOL, - DIR_ONE, - DIR_ONE_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - getBucketLayout()); - writeDirToOm(reconOMMetadataManager, - (DIR_ONE + OM_KEY_PREFIX + DIR_TWO + OM_KEY_PREFIX), - BUCKET_ONE, - VOL, - DIR_TWO, - DIR_TWO_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - getBucketLayout()); - writeDirToOm(reconOMMetadataManager, - (DIR_ONE + OM_KEY_PREFIX + DIR_THREE + OM_KEY_PREFIX), - BUCKET_ONE, - VOL, - DIR_THREE, - DIR_THREE_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - getBucketLayout()); - writeDirToOm(reconOMMetadataManager, - (DIR_ONE + OM_KEY_PREFIX + DIR_FOUR + OM_KEY_PREFIX), - BUCKET_ONE, - VOL, - DIR_FOUR, - DIR_FOUR_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - getBucketLayout()); - writeDirToOm(reconOMMetadataManager, - (DIR_FIVE + OM_KEY_PREFIX), - BUCKET_THREE, - VOL_TWO, - DIR_FIVE, - DIR_FIVE_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - getBucketLayout()); - - // write all keys - writeKeyToOm(reconOMMetadataManager, - KEY_ONE, - BUCKET_ONE, - VOL, - FILE_ONE, - KEY_ONE_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - KEY_ONE_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_TWO, - BUCKET_ONE, - VOL, - FILE_TWO, - KEY_TWO_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - KEY_TWO_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_THREE, - BUCKET_ONE, - VOL, - FILE_THREE, - KEY_THREE_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - KEY_THREE_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_FOUR, - BUCKET_TWO, - VOL, - FILE_FOUR, - KEY_FOUR_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_TWO_OBJECT_ID, - VOL_OBJECT_ID, - KEY_FOUR_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_FIVE, - BUCKET_TWO, - VOL, - FILE_FIVE, - KEY_FIVE_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_TWO_OBJECT_ID, - VOL_OBJECT_ID, - KEY_FIVE_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_SIX, - BUCKET_ONE, - VOL, - FILE_SIX, - KEY_SIX_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - KEY_SIX_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_EIGHT, - BUCKET_THREE, - VOL_TWO, - FILE_EIGHT, - KEY_EIGHT_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - KEY_EIGHT_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_NINE, - BUCKET_THREE, - VOL_TWO, - FILE_NINE, - KEY_NINE_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - KEY_NINE_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_TEN, - BUCKET_THREE, - VOL_TWO, - FILE_TEN, - KEY_TEN_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - KEY_TEN_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_ELEVEN, - BUCKET_FOUR, - VOL_TWO, - FILE_ELEVEN, - KEY_ELEVEN_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_FOUR_OBJECT_ID, - VOL_TWO_OBJECT_ID, - KEY_ELEVEN_SIZE, - getBucketLayout()); - } - - private void populateVolumeThree() throws IOException { - - writeDirToOm(reconOMMetadataManager, - (DIR_SIX + OM_KEY_PREFIX), - BUCKET_FIVE, - VOL_THREE, - DIR_SIX, - DIR_SIX_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_FIVE_OBJECT_ID, - VOL_THREE_OBJECT_ID, - getBucketLayout()); - - writeDirToOm(reconOMMetadataManager, - (DIR_SIX + OM_KEY_PREFIX + DIR_SEVEN + OM_KEY_PREFIX), - BUCKET_FIVE, - VOL_THREE, - DIR_SEVEN, - DIR_SEVEN_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_FIVE_OBJECT_ID, - VOL_THREE_OBJECT_ID, - getBucketLayout()); - - writeKeyToOm(reconOMMetadataManager, - KEY_TWELVE, - BUCKET_FIVE, - VOL_THREE, - FILE_TWELVE, - FILE_TWELVE_OBJECT_ID, - BUCKET_FIVE_OBJECT_ID, - BUCKET_FIVE_OBJECT_ID, - VOL_THREE_OBJECT_ID, - KEY_TWELVE_SIZE, - getBucketLayout(), RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), - Time.now(), true); - - writeKeyToOm(reconOMMetadataManager, - KEY_THIRTEEN, - BUCKET_FIVE, - VOL_THREE, - FILE_THIRTEEN, - FILE_THIRTEEN_OBJECT_ID, - DIR_SIX_OBJECT_ID, - BUCKET_FIVE_OBJECT_ID, - VOL_THREE_OBJECT_ID, - KEY_THIRTEEN_SIZE, - getBucketLayout(), RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), - Time.now(), true); - - writeKeyToOm(reconOMMetadataManager, - KEY_FOURTEEN, - BUCKET_FIVE, - VOL_THREE, - FILE_FOURTEEN, - FILE_FOURTEEN_OBJECT_ID, - DIR_SEVEN_OBJECT_ID, - BUCKET_FIVE_OBJECT_ID, - VOL_THREE_OBJECT_ID, - KEY_FOURTEEN_SIZE, - getBucketLayout(), RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), - Time.now(), true); - } - - /** - * Create a new OM Metadata manager instance with one user, one vol, and two - * buckets. - * @throws IOException ioEx - */ - private static OMMetadataManager initializeNewOmMetadataManager( - File omDbDir, OzoneConfiguration omConfiguration) - throws IOException { - omConfiguration.set(OZONE_OM_DB_DIRS, - omDbDir.getAbsolutePath()); - omConfiguration.set(OMConfigKeys - .OZONE_OM_ENABLE_FILESYSTEM_PATHS, "true"); - setConfiguration(omConfiguration); - OMMetadataManager omMetadataManager = new OmMetadataManagerImpl( - omConfiguration, null); - - String volumeKey = omMetadataManager.getVolumeKey(VOL); - OmVolumeArgs args = - OmVolumeArgs.newBuilder() - .setObjectID(VOL_OBJECT_ID) - .setVolume(VOL) - .setAdminName(TEST_USER) - .setOwnerName(TEST_USER) - .setQuotaInBytes(VOL_QUOTA) - .build(); - - String volume2Key = omMetadataManager.getVolumeKey(VOL_TWO); - OmVolumeArgs args2 = - OmVolumeArgs.newBuilder() - .setObjectID(VOL_TWO_OBJECT_ID) - .setVolume(VOL_TWO) - .setAdminName(TEST_USER) - .setOwnerName(TEST_USER) - .setQuotaInBytes(VOL_TWO_QUOTA) - .build(); - - String volume3Key = omMetadataManager.getVolumeKey(VOL_THREE); - OmVolumeArgs args3 = - OmVolumeArgs.newBuilder() - .setObjectID(VOL_THREE_OBJECT_ID) - .setVolume(VOL_THREE) - .setAdminName(TEST_USER) - .setOwnerName(TEST_USER) - .setQuotaInBytes(VOL_THREE_QUOTA) - .build(); - - omMetadataManager.getVolumeTable().put(volumeKey, args); - omMetadataManager.getVolumeTable().put(volume2Key, args2); - omMetadataManager.getVolumeTable().put(volume3Key, args3); - - OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() - .setVolumeName(VOL) - .setBucketName(BUCKET_ONE) - .setObjectID(BUCKET_ONE_OBJECT_ID) - .setQuotaInBytes(BUCKET_ONE_QUOTA) - .setBucketLayout(getBucketLayout()) - .build(); - - OmBucketInfo bucketInfo2 = OmBucketInfo.newBuilder() - .setVolumeName(VOL) - .setBucketName(BUCKET_TWO) - .setObjectID(BUCKET_TWO_OBJECT_ID) - .setQuotaInBytes(BUCKET_TWO_QUOTA) - .setBucketLayout(getBucketLayout()) - .build(); - - OmBucketInfo bucketInfo3 = OmBucketInfo.newBuilder() - .setVolumeName(VOL_TWO) - .setBucketName(BUCKET_THREE) - .setObjectID(BUCKET_THREE_OBJECT_ID) - .setQuotaInBytes(BUCKET_THREE_QUOTA) - .setBucketLayout(getBucketLayout()) - .build(); - - OmBucketInfo bucketInfo4 = OmBucketInfo.newBuilder() - .setVolumeName(VOL_TWO) - .setBucketName(BUCKET_FOUR) - .setObjectID(BUCKET_FOUR_OBJECT_ID) - .setQuotaInBytes(BUCKET_FOUR_QUOTA) - .setBucketLayout(getBucketLayout()) - .build(); - - OmBucketInfo bucketInfo5 = OmBucketInfo.newBuilder() - .setVolumeName(VOL_THREE) - .setBucketName(BUCKET_FIVE) - .setObjectID(BUCKET_FIVE_OBJECT_ID) - .setQuotaInBytes(BUCKET_FIVE_QUOTA) - .setBucketLayout(getBucketLayout()) - .build(); - - String bucketKey = omMetadataManager.getBucketKey( - bucketInfo.getVolumeName(), bucketInfo.getBucketName()); - String bucketKey2 = omMetadataManager.getBucketKey( - bucketInfo2.getVolumeName(), bucketInfo2.getBucketName()); - String bucketKey3 = omMetadataManager.getBucketKey( - bucketInfo3.getVolumeName(), bucketInfo3.getBucketName()); - String bucketKey4 = omMetadataManager.getBucketKey( - bucketInfo4.getVolumeName(), bucketInfo4.getBucketName()); - String bucketKey5 = omMetadataManager.getBucketKey( - bucketInfo5.getVolumeName(), bucketInfo5.getBucketName()); - - omMetadataManager.getBucketTable().put(bucketKey, bucketInfo); - omMetadataManager.getBucketTable().put(bucketKey2, bucketInfo2); - omMetadataManager.getBucketTable().put(bucketKey3, bucketInfo3); - omMetadataManager.getBucketTable().put(bucketKey4, bucketInfo4); - omMetadataManager.getBucketTable().put(bucketKey5, bucketInfo5); - - return omMetadataManager; - } - - private void setUpMultiBlockKey() throws IOException { - OmKeyLocationInfoGroup locationInfoGroup = - getLocationInfoGroup1(); - - // add the multi-block key to Recon's OM - writeKeyToOm(reconOMMetadataManager, - MULTI_BLOCK_KEY, - BUCKET_ONE, - VOL, - MULTI_BLOCK_FILE, - MULTI_BLOCK_KEY_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup), - getBucketLayout(), - KEY_SEVEN_SIZE); - } - - private OmKeyLocationInfoGroup getLocationInfoGroup1() { - List<OmKeyLocationInfo> locationInfoList = new ArrayList<>(); - BlockID block1 = new BlockID(CONTAINER_ONE_ID, 0L); - BlockID block2 = new BlockID(CONTAINER_TWO_ID, 0L); - BlockID block3 = new BlockID(CONTAINER_THREE_ID, 0L); - - OmKeyLocationInfo location1 = new OmKeyLocationInfo.Builder() - .setBlockID(block1) - .setLength(BLOCK_ONE_LENGTH) - .build(); - OmKeyLocationInfo location2 = new OmKeyLocationInfo.Builder() - .setBlockID(block2) - .setLength(BLOCK_TWO_LENGTH) - .build(); - OmKeyLocationInfo location3 = new OmKeyLocationInfo.Builder() - .setBlockID(block3) - .setLength(BLOCK_THREE_LENGTH) - .build(); - locationInfoList.add(location1); - locationInfoList.add(location2); - locationInfoList.add(location3); - - return new OmKeyLocationInfoGroup(0L, locationInfoList); - } - - /** - * Testing the following case. - * vol - * / \ - * bucket1 bucket2 - * / \ / \ - * file1 dir1 file4 file5 - * / \ \ \ - * dir2 dir3 dir4 file7 - * / \ \ - * file2 file3 file6 - * ---------------------------------------- - * vol2 - * / \ - * bucket3 bucket4 - * / \ / - * file8 dir5 file11 - * / \ - * file9 file10 - * Write these keys to OM and - * replicate them. - */ - private OmKeyLocationInfoGroup getLocationInfoGroup2() { - List<OmKeyLocationInfo> locationInfoList = new ArrayList<>(); - BlockID block4 = new BlockID(CONTAINER_FOUR_ID, 0L); - BlockID block5 = new BlockID(CONTAINER_FIVE_ID, 0L); - BlockID block6 = new BlockID(CONTAINER_SIX_ID, 0L); - - OmKeyLocationInfo location4 = new OmKeyLocationInfo.Builder() - .setBlockID(block4) - .setLength(BLOCK_FOUR_LENGTH) - .build(); - OmKeyLocationInfo location5 = new OmKeyLocationInfo.Builder() - .setBlockID(block5) - .setLength(BLOCK_FIVE_LENGTH) - .build(); - OmKeyLocationInfo location6 = new OmKeyLocationInfo.Builder() - .setBlockID(block6) - .setLength(BLOCK_SIX_LENGTH) - .build(); - locationInfoList.add(location4); - locationInfoList.add(location5); - locationInfoList.add(location6); - return new OmKeyLocationInfoGroup(0L, locationInfoList); - - } - - @SuppressWarnings("checkstyle:MethodLength") - private void setUpMultiBlockReplicatedKeys() throws IOException { - OmKeyLocationInfoGroup locationInfoGroup1 = - getLocationInfoGroup1(); - OmKeyLocationInfoGroup locationInfoGroup2 = - getLocationInfoGroup2(); - - //vol/bucket1/file1 - writeKeyToOm(reconOMMetadataManager, - KEY_ONE, - BUCKET_ONE, - VOL, - FILE_ONE, - KEY_ONE_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getBucketLayout(), - KEY_ONE_SIZE); - - //vol/bucket1/dir1/dir2/file2 - writeKeyToOm(reconOMMetadataManager, - KEY_TWO, - BUCKET_ONE, - VOL, - FILE_TWO, - KEY_TWO_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getBucketLayout(), - KEY_TWO_SIZE); - - //vol/bucket1/dir1/dir3/file3 - writeKeyToOm(reconOMMetadataManager, - KEY_THREE, - BUCKET_ONE, - VOL, - FILE_THREE, - KEY_THREE_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getBucketLayout(), - KEY_THREE_SIZE); - - //vol/bucket2/file4 - writeKeyToOm(reconOMMetadataManager, - KEY_FOUR, - BUCKET_TWO, - VOL, - FILE_FOUR, - KEY_FOUR_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_TWO_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getBucketLayout(), - KEY_FOUR_SIZE); - - //vol/bucket2/file5 - writeKeyToOm(reconOMMetadataManager, - KEY_FIVE, - BUCKET_TWO, - VOL, - FILE_FIVE, - KEY_FIVE_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_TWO_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getBucketLayout(), - KEY_FIVE_SIZE); - - //vol/bucket1/dir1/dir4/file6 - writeKeyToOm(reconOMMetadataManager, - KEY_SIX, - BUCKET_ONE, - VOL, - FILE_SIX, - KEY_SIX_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getBucketLayout(), - KEY_SIX_SIZE); - - //vol/bucket1/dir1/file7 - writeKeyToOm(reconOMMetadataManager, - KEY_SEVEN, - BUCKET_ONE, - VOL, - FILE_SEVEN, - KEY_SEVEN_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getBucketLayout(), - KEY_SEVEN_SIZE); - - //vol2/bucket3/file8 - writeKeyToOm(reconOMMetadataManager, - KEY_EIGHT, - BUCKET_THREE, - VOL_TWO, - FILE_EIGHT, - KEY_EIGHT_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getBucketLayout(), - KEY_EIGHT_SIZE); - - //vol2/bucket3/dir5/file9 - writeKeyToOm(reconOMMetadataManager, - KEY_NINE, - BUCKET_THREE, - VOL_TWO, - FILE_NINE, - KEY_NINE_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getBucketLayout(), - KEY_NINE_SIZE); - - //vol2/bucket3/dir5/file10 - writeKeyToOm(reconOMMetadataManager, - KEY_TEN, - BUCKET_THREE, - VOL_TWO, - FILE_TEN, - KEY_TEN_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getBucketLayout(), - KEY_TEN_SIZE); - - //vol2/bucket4/file11 - writeKeyToOm(reconOMMetadataManager, - KEY_ELEVEN, - BUCKET_FOUR, - VOL_TWO, - FILE_ELEVEN, - KEY_ELEVEN_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_FOUR_OBJECT_ID, - VOL_TWO_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getBucketLayout(), - KEY_ELEVEN_SIZE); - } - - /** - * Generate a set of mock container replica with a size of - * replication factor for container. - * @param replicationFactor number of replica - * @param containerID the container replicated based upon - * @return a set of container replica for testing - */ - private static Set<ContainerReplica> generateMockContainerReplicas( - int replicationFactor, ContainerID containerID) { - Set<ContainerReplica> result = new HashSet<>(); - for (int i = 0; i < replicationFactor; ++i) { - DatanodeDetails randomDatanode = randomDatanodeDetails(); - ContainerReplica replica = new ContainerReplica.ContainerReplicaBuilder() - .setContainerID(containerID) - .setContainerState(State.OPEN) - .setDatanodeDetails(randomDatanode) - .build(); - result.add(replica); - } - return result; - } - - private static ReconStorageContainerManagerFacade getMockReconSCM() - throws ContainerNotFoundException { - ReconStorageContainerManagerFacade reconSCM = - mock(ReconStorageContainerManagerFacade.class); - ContainerManager containerManager = mock(ContainerManager.class); - - // Container 1 is 3-way replicated - ContainerID containerID1 = ContainerID.valueOf(CONTAINER_ONE_ID); - Set<ContainerReplica> containerReplicas1 = generateMockContainerReplicas( - CONTAINER_ONE_REPLICA_COUNT, containerID1); - when(containerManager.getContainerReplicas(containerID1)) - .thenReturn(containerReplicas1); - - // Container 2 is under replicated with 2 replica - ContainerID containerID2 = ContainerID.valueOf(CONTAINER_TWO_ID); - Set<ContainerReplica> containerReplicas2 = generateMockContainerReplicas( - CONTAINER_TWO_REPLICA_COUNT, containerID2); - when(containerManager.getContainerReplicas(containerID2)) - .thenReturn(containerReplicas2); - - // Container 3 is over replicated with 4 replica - ContainerID containerID3 = ContainerID.valueOf(CONTAINER_THREE_ID); - Set<ContainerReplica> containerReplicas3 = generateMockContainerReplicas( - CONTAINER_THREE_REPLICA_COUNT, containerID3); - when(containerManager.getContainerReplicas(containerID3)) - .thenReturn(containerReplicas3); - - // Container 4 is replicated with 5 replica - ContainerID containerID4 = ContainerID.valueOf(CONTAINER_FOUR_ID); - Set<ContainerReplica> containerReplicas4 = generateMockContainerReplicas( - CONTAINER_FOUR_REPLICA_COUNT, containerID4); - when(containerManager.getContainerReplicas(containerID4)) - .thenReturn(containerReplicas4); - - // Container 5 is replicated with 2 replica - ContainerID containerID5 = ContainerID.valueOf(CONTAINER_FIVE_ID); - Set<ContainerReplica> containerReplicas5 = generateMockContainerReplicas( - CONTAINER_FIVE_REPLICA_COUNT, containerID5); - when(containerManager.getContainerReplicas(containerID5)) - .thenReturn(containerReplicas5); - - // Container 6 is replicated with 3 replica - ContainerID containerID6 = ContainerID.valueOf(CONTAINER_SIX_ID); - Set<ContainerReplica> containerReplicas6 = generateMockContainerReplicas( - CONTAINER_SIX_REPLICA_COUNT, containerID6); - when(containerManager.getContainerReplicas(containerID6)) - .thenReturn(containerReplicas6); - - when(reconSCM.getContainerManager()).thenReturn(containerManager); - ReconNodeManager mockReconNodeManager = mock(ReconNodeManager.class); - when(mockReconNodeManager.getStats()).thenReturn(getMockSCMRootStat()); - when(reconSCM.getScmNodeManager()).thenReturn(mockReconNodeManager); - return reconSCM; - } - - private static BucketLayout getBucketLayout() { - return BucketLayout.LEGACY; - } - - private static SCMNodeStat getMockSCMRootStat() { - return new SCMNodeStat(ROOT_QUOTA, ROOT_DATA_SIZE, - ROOT_QUOTA - ROOT_DATA_SIZE, 0, ROOT_QUOTA - ROOT_DATA_SIZE - 1, 0); - } -}
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithOBSAndLegacy.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithOBSAndLegacy.java deleted file mode 100644 index ecb08c1..0000000 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithOBSAndLegacy.java +++ /dev/null
@@ -1,1452 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.recon.api; - -import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; -import static org.apache.hadoop.ozone.om.helpers.QuotaUtil.getReplicatedSize; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProvider; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import javax.ws.rs.core.Response; -import org.apache.hadoop.hdds.client.BlockID; -import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.StorageType; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos; -import org.apache.hadoop.hdds.scm.container.ContainerID; -import org.apache.hadoop.hdds.scm.container.ContainerManager; -import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; -import org.apache.hadoop.hdds.scm.container.ContainerReplica; -import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat; -import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; -import org.apache.hadoop.ozone.OmUtils; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; -import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; -import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; -import org.apache.hadoop.ozone.recon.ReconConstants; -import org.apache.hadoop.ozone.recon.ReconTestInjector; -import org.apache.hadoop.ozone.recon.ReconUtils; -import org.apache.hadoop.ozone.recon.api.handlers.BucketHandler; -import org.apache.hadoop.ozone.recon.api.handlers.EntityHandler; -import org.apache.hadoop.ozone.recon.api.types.BucketObjectDBInfo; -import org.apache.hadoop.ozone.recon.api.types.DUResponse; -import org.apache.hadoop.ozone.recon.api.types.EntityType; -import org.apache.hadoop.ozone.recon.api.types.FileSizeDistributionResponse; -import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse; -import org.apache.hadoop.ozone.recon.api.types.QuotaUsageResponse; -import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; -import org.apache.hadoop.ozone.recon.api.types.VolumeObjectDBInfo; -import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; -import org.apache.hadoop.ozone.recon.scm.ReconNodeManager; -import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; -import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; -import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; -import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; -import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; -import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithLegacy; -import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithOBS; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -/** - * Tests the NSSummary REST APIs within the context of an Object Store (OBS) layout, - * as well as Legacy layout buckets with FileSystemPaths disabled. The tests aim to - * validate API responses for buckets that follow the flat hierarchy model typical - * of OBS layouts. - * <p> - * The test environment simulates a simple object storage structure with volumes - * containing buckets, which in turn contain files. Specifically, it includes: - * - Two OBS layout buckets (bucket1 and bucket2) under 'vol', each containing - * multiple files. - * - Two Legacy layout buckets (bucket3 and bucket4) under 'vol2', with 'bucket4' - * the fileSystemEnabled flag set to false for these legacy buckets. - * <p> - * The directory structure for testing is as follows: - * . - * └── vol - * ├── bucket1 (OBS) - * │ ├── KEY_ONE - * │ ├── KEY_TWO - * │ └── KEY_THREE - * └── bucket2 (OBS) - * ├── KEY_FOUR - * └── KEY_FIVE - * └── vol2 - * ├── bucket3 (Legacy) - * │ ├── KEY_EIGHT - * │ ├── KEY_NINE - * │ └── KEY_TEN - * └── bucket4 (Legacy) - * └── KEY_ELEVEN - */ -public class TestNSSummaryEndpointWithOBSAndLegacy extends NSSummaryTests { - @TempDir - private Path temporaryFolder; - - private ReconOMMetadataManager reconOMMetadataManager; - private ReconNamespaceSummaryManager reconNamespaceSummaryManager; - private NSSummaryEndpoint nsSummaryEndpoint; - - private static final String TEST_PATH_UTILITY = - "/vol1/buck1/a/b/c/d/e/file1.txt"; - private static final String PARENT_DIR = "vol1/buck1/a/b/c/d/e"; - private static final String[] TEST_NAMES = - new String[]{"vol1", "buck1", "a", "b", "c", "d", "e", "file1.txt"}; - private static final String TEST_KEY_NAMES = "a/b/c/d/e/file1.txt"; - - // Object names - private static final String VOL = "vol"; - private static final String VOL_TWO = "vol2"; - private static final String BUCKET_ONE = "bucket1"; - private static final String BUCKET_TWO = "bucket2"; - private static final String BUCKET_THREE = "bucket3"; - private static final String BUCKET_FOUR = "bucket4"; - private static final String KEY_ONE = "file1"; - private static final String KEY_TWO = "////file2"; - private static final String KEY_THREE = "file3///"; - private static final String KEY_FOUR = "file4"; - private static final String KEY_FIVE = "_//////"; - private static final String KEY_EIGHT = "file8"; - private static final String KEY_NINE = "//////"; - private static final String KEY_TEN = "///__file10"; - private static final String KEY_ELEVEN = "////file11"; - private static final String MULTI_BLOCK_FILE = KEY_THREE; - - private static final long PARENT_OBJECT_ID_ZERO = 0L; - private static final long VOL_OBJECT_ID = 0L; - private static final long VOL_TWO_OBJECT_ID = 14L; - private static final long BUCKET_ONE_OBJECT_ID = 1L; - private static final long BUCKET_TWO_OBJECT_ID = 2L; - private static final long BUCKET_THREE_OBJECT_ID = 15L; - private static final long BUCKET_FOUR_OBJECT_ID = 16L; - private static final long KEY_ONE_OBJECT_ID = 3L; - private static final long KEY_TWO_OBJECT_ID = 5L; - private static final long KEY_THREE_OBJECT_ID = 8L; - private static final long KEY_FOUR_OBJECT_ID = 6L; - private static final long KEY_FIVE_OBJECT_ID = 9L; - private static final long KEY_EIGHT_OBJECT_ID = 17L; - private static final long KEY_NINE_OBJECT_ID = 19L; - private static final long KEY_TEN_OBJECT_ID = 20L; - private static final long KEY_ELEVEN_OBJECT_ID = 21L; - private static final long MULTI_BLOCK_KEY_OBJECT_ID = 13L; - - // container IDs - private static final long CONTAINER_ONE_ID = 1L; - private static final long CONTAINER_TWO_ID = 2L; - private static final long CONTAINER_THREE_ID = 3L; - private static final long CONTAINER_FOUR_ID = 4L; - private static final long CONTAINER_FIVE_ID = 5L; - private static final long CONTAINER_SIX_ID = 6L; - - // replication factors - private static final int CONTAINER_ONE_REPLICA_COUNT = 3; - private static final int CONTAINER_TWO_REPLICA_COUNT = 2; - private static final int CONTAINER_THREE_REPLICA_COUNT = 4; - private static final int CONTAINER_FOUR_REPLICA_COUNT = 5; - private static final int CONTAINER_FIVE_REPLICA_COUNT = 2; - private static final int CONTAINER_SIX_REPLICA_COUNT = 3; - - // block lengths - private static final long BLOCK_ONE_LENGTH = 1000L; - private static final long BLOCK_TWO_LENGTH = 2000L; - private static final long BLOCK_THREE_LENGTH = 3000L; - private static final long BLOCK_FOUR_LENGTH = 4000L; - private static final long BLOCK_FIVE_LENGTH = 5000L; - private static final long BLOCK_SIX_LENGTH = 6000L; - - // data size in bytes - private static final long FILE_ONE_SIZE = 500L; // bin 0 - private static final long FILE_TWO_SIZE = OzoneConsts.KB + 1; // bin 1 - private static final long FILE_THREE_SIZE = 4 * OzoneConsts.KB + 1; // bin 3 - private static final long FILE_FOUR_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 - private static final long FILE_FIVE_SIZE = 100L; // bin 0 - private static final long FILE_EIGHT_SIZE = OzoneConsts.KB + 1; // bin 1 - private static final long FILE_NINE_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 - private static final long FILE_TEN_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 - private static final long FILE_ELEVEN_SIZE = OzoneConsts.KB + 1; // bin 1 - - private static final long FILE1_SIZE_WITH_REPLICA = - getReplicatedSize(FILE_ONE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE2_SIZE_WITH_REPLICA = - getReplicatedSize(FILE_TWO_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE3_SIZE_WITH_REPLICA = - getReplicatedSize(FILE_THREE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE4_SIZE_WITH_REPLICA = - getReplicatedSize(FILE_FOUR_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE5_SIZE_WITH_REPLICA = - getReplicatedSize(FILE_FIVE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - - private static final long FILE8_SIZE_WITH_REPLICA = - getReplicatedSize(FILE_EIGHT_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE9_SIZE_WITH_REPLICA = - getReplicatedSize(FILE_NINE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE10_SIZE_WITH_REPLICA = - getReplicatedSize(FILE_TEN_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE11_SIZE_WITH_REPLICA = - getReplicatedSize(FILE_ELEVEN_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - - private static final long MULTI_BLOCK_KEY_SIZE_WITH_REPLICA - = FILE3_SIZE_WITH_REPLICA; - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_ROOT - = FILE1_SIZE_WITH_REPLICA - + FILE2_SIZE_WITH_REPLICA - + FILE3_SIZE_WITH_REPLICA - + FILE4_SIZE_WITH_REPLICA - + FILE5_SIZE_WITH_REPLICA - + FILE8_SIZE_WITH_REPLICA - + FILE9_SIZE_WITH_REPLICA - + FILE10_SIZE_WITH_REPLICA - + FILE11_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_VOL - = FILE1_SIZE_WITH_REPLICA - + FILE2_SIZE_WITH_REPLICA - + FILE3_SIZE_WITH_REPLICA - + FILE4_SIZE_WITH_REPLICA - + FILE5_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET1 - = FILE1_SIZE_WITH_REPLICA - + FILE2_SIZE_WITH_REPLICA - + FILE3_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET3 - = FILE8_SIZE_WITH_REPLICA + - FILE9_SIZE_WITH_REPLICA + - FILE10_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_KEY - = FILE4_SIZE_WITH_REPLICA; - - // quota in bytes - private static final long ROOT_QUOTA = 2 * (2 * OzoneConsts.MB); - private static final long VOL_QUOTA = 2 * OzoneConsts.MB; - private static final long VOL_TWO_QUOTA = 2 * OzoneConsts.MB; - private static final long BUCKET_ONE_QUOTA = OzoneConsts.MB; - private static final long BUCKET_TWO_QUOTA = OzoneConsts.MB; - private static final long BUCKET_THREE_QUOTA = OzoneConsts.MB; - private static final long BUCKET_FOUR_QUOTA = OzoneConsts.MB; - - // mock client's path requests - private static final String TEST_USER = "TestUser"; - private static final String ROOT_PATH = "/"; - private static final String VOL_PATH = ROOT_PATH + VOL; - private static final String VOL_TWO_PATH = ROOT_PATH + VOL_TWO; - private static final String BUCKET_ONE_PATH = - ROOT_PATH + VOL + ROOT_PATH + BUCKET_ONE; - private static final String BUCKET_TWO_PATH = - ROOT_PATH + VOL + ROOT_PATH + BUCKET_TWO; - private static final String BUCKET_THREE_PATH = - ROOT_PATH + VOL_TWO + ROOT_PATH + BUCKET_THREE; - private static final String BUCKET_FOUR_PATH = - ROOT_PATH + VOL_TWO + ROOT_PATH + BUCKET_FOUR; - private static final String KEY_ONE_PATH = - ROOT_PATH + VOL + ROOT_PATH + BUCKET_ONE + ROOT_PATH + KEY_ONE; - private static final String KEY_TWO_PATH = - ROOT_PATH + VOL + ROOT_PATH + BUCKET_ONE + ROOT_PATH + KEY_TWO; - private static final String KEY_FIVE_PATH = - ROOT_PATH + VOL + ROOT_PATH + BUCKET_TWO + ROOT_PATH + KEY_FIVE; - private static final String KEY_EIGHT_PATH = - ROOT_PATH + VOL_TWO + ROOT_PATH + BUCKET_THREE + ROOT_PATH + KEY_EIGHT; - private static final String KEY_ELEVEN_PATH = - ROOT_PATH + VOL_TWO + ROOT_PATH + BUCKET_FOUR + ROOT_PATH + KEY_ELEVEN; - private static final String KEY4_PATH = - ROOT_PATH + VOL + ROOT_PATH + BUCKET_TWO + ROOT_PATH + KEY_FOUR; - private static final String MULTI_BLOCK_KEY_PATH = - ROOT_PATH + VOL + ROOT_PATH + BUCKET_ONE + ROOT_PATH + KEY_THREE; - private static final String INVALID_PATH = "/vol/path/not/found"; - - // some expected answers - private static final long ROOT_DATA_SIZE = - FILE_ONE_SIZE + FILE_TWO_SIZE + FILE_THREE_SIZE + FILE_FOUR_SIZE + - FILE_FIVE_SIZE + FILE_EIGHT_SIZE + FILE_NINE_SIZE + FILE_TEN_SIZE + - FILE_ELEVEN_SIZE; - private static final long VOL_DATA_SIZE = FILE_ONE_SIZE + FILE_TWO_SIZE + - FILE_THREE_SIZE + FILE_FOUR_SIZE + FILE_FIVE_SIZE; - - private static final long VOL_TWO_DATA_SIZE = - FILE_EIGHT_SIZE + FILE_NINE_SIZE + FILE_TEN_SIZE + FILE_ELEVEN_SIZE; - - private static final long BUCKET_ONE_DATA_SIZE = FILE_ONE_SIZE + - FILE_TWO_SIZE + - FILE_THREE_SIZE; - - private static final long BUCKET_TWO_DATA_SIZE = - FILE_FOUR_SIZE + FILE_FIVE_SIZE; - - private static final long BUCKET_THREE_DATA_SIZE = - FILE_EIGHT_SIZE + FILE_NINE_SIZE + FILE_TEN_SIZE; - - private static final long BUCKET_FOUR_DATA_SIZE = FILE_ELEVEN_SIZE; - - @BeforeEach - public void setUp() throws Exception { - OzoneConfiguration conf = new OzoneConfiguration(); - // By setting this config our Legacy buckets will behave like OBS buckets. - conf.set(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, "false"); - OMMetadataManager omMetadataManager = initializeNewOmMetadataManager( - Files.createDirectory(temporaryFolder.resolve( - "JunitOmDBDir")).toFile(), conf); - OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = - getMockOzoneManagerServiceProvider(); - reconOMMetadataManager = getTestReconOmMetadataManager(omMetadataManager, - Files.createDirectory(temporaryFolder.resolve( - "omMetadatDir")).toFile()); - - ReconTestInjector reconTestInjector = - new ReconTestInjector.Builder(temporaryFolder.toFile()) - .withReconOm(reconOMMetadataManager) - .withOmServiceProvider(ozoneManagerServiceProvider) - .withReconSqlDb() - .withContainerDB() - .addBinding(OzoneStorageContainerManager.class, - getMockReconSCM()) - .addBinding(StorageContainerServiceProvider.class, - mock(StorageContainerServiceProviderImpl.class)) - .addBinding(NSSummaryEndpoint.class) - .build(); - reconNamespaceSummaryManager = - reconTestInjector.getInstance(ReconNamespaceSummaryManager.class); - nsSummaryEndpoint = reconTestInjector.getInstance(NSSummaryEndpoint.class); - - // populate OM DB and reprocess into Recon RocksDB - populateOMDB(); - NSSummaryTaskWithOBS nsSummaryTaskWithOBS = - new NSSummaryTaskWithOBS(reconNamespaceSummaryManager, - reconOMMetadataManager, 10, 5, 20, 2000); - nsSummaryTaskWithOBS.reprocessWithOBS(reconOMMetadataManager); - NSSummaryTaskWithLegacy nsSummaryTaskWithLegacy = - new NSSummaryTaskWithLegacy(reconNamespaceSummaryManager, - reconOMMetadataManager, conf, 10); - nsSummaryTaskWithLegacy.reprocessWithLegacy(reconOMMetadataManager); - } - - @Test - public void testUtility() { - String[] names = EntityHandler.parseRequestPath(TEST_PATH_UTILITY); - assertArrayEquals(TEST_NAMES, names); - String keyName = BucketHandler.getKeyName(names); - assertEquals(TEST_KEY_NAMES, keyName); - String subpath = BucketHandler.buildSubpath(PARENT_DIR, "file1.txt"); - assertEquals(TEST_PATH_UTILITY, subpath); - } - - @Test - public void testGetBasicInfoRoot() throws Exception { - // Test root basics - Response rootResponse = nsSummaryEndpoint.getBasicInfo(ROOT_PATH); - NamespaceSummaryResponse rootResponseObj = - (NamespaceSummaryResponse) rootResponse.getEntity(); - assertEquals(EntityType.ROOT, rootResponseObj.getEntityType()); - assertEquals(2, rootResponseObj.getCountStats().getNumVolume()); - assertEquals(4, rootResponseObj.getCountStats().getNumBucket()); - assertEquals(9, rootResponseObj.getCountStats().getNumTotalKey()); - } - - @Test - public void testGetBasicInfoVol() throws Exception { - // Test volume basics - Response volResponse = nsSummaryEndpoint.getBasicInfo(VOL_PATH); - NamespaceSummaryResponse volResponseObj = - (NamespaceSummaryResponse) volResponse.getEntity(); - assertEquals(EntityType.VOLUME, - volResponseObj.getEntityType()); - assertEquals(2, volResponseObj.getCountStats().getNumBucket()); - assertEquals(5, volResponseObj.getCountStats().getNumTotalKey()); - assertEquals(TEST_USER, ((VolumeObjectDBInfo) volResponseObj. - getObjectDBInfo()).getAdmin()); - assertEquals(TEST_USER, ((VolumeObjectDBInfo) volResponseObj. - getObjectDBInfo()).getOwner()); - assertEquals(VOL, volResponseObj.getObjectDBInfo().getName()); - assertEquals(2097152, volResponseObj.getObjectDBInfo().getQuotaInBytes()); - assertEquals(-1, volResponseObj.getObjectDBInfo().getQuotaInNamespace()); - } - - @Test - public void testGetBasicInfoVolTwo() throws Exception { - // Test volume 2's basics - Response volTwoResponse = nsSummaryEndpoint.getBasicInfo(VOL_TWO_PATH); - NamespaceSummaryResponse volTwoResponseObj = - (NamespaceSummaryResponse) volTwoResponse.getEntity(); - assertEquals(EntityType.VOLUME, - volTwoResponseObj.getEntityType()); - assertEquals(2, volTwoResponseObj.getCountStats().getNumBucket()); - assertEquals(4, volTwoResponseObj.getCountStats().getNumTotalKey()); - assertEquals(TEST_USER, ((VolumeObjectDBInfo) volTwoResponseObj. - getObjectDBInfo()).getAdmin()); - assertEquals(TEST_USER, ((VolumeObjectDBInfo) volTwoResponseObj. - getObjectDBInfo()).getOwner()); - assertEquals(VOL_TWO, volTwoResponseObj.getObjectDBInfo().getName()); - assertEquals(2097152, - volTwoResponseObj.getObjectDBInfo().getQuotaInBytes()); - assertEquals(-1, volTwoResponseObj.getObjectDBInfo().getQuotaInNamespace()); - } - - @Test - public void testGetBasicInfoBucketOne() throws Exception { - // Test bucket 1's basics - Response bucketOneResponse = - nsSummaryEndpoint.getBasicInfo(BUCKET_ONE_PATH); - NamespaceSummaryResponse bucketOneObj = - (NamespaceSummaryResponse) bucketOneResponse.getEntity(); - assertEquals(EntityType.BUCKET, bucketOneObj.getEntityType()); - assertEquals(3, bucketOneObj.getCountStats().getNumTotalKey()); - assertEquals(VOL, - ((BucketObjectDBInfo) bucketOneObj.getObjectDBInfo()).getVolumeName()); - assertEquals(StorageType.DISK, - ((BucketObjectDBInfo) - bucketOneObj.getObjectDBInfo()).getStorageType()); - assertEquals(getOBSBucketLayout(), - ((BucketObjectDBInfo) - bucketOneObj.getObjectDBInfo()).getBucketLayout()); - assertEquals(BUCKET_ONE, - ((BucketObjectDBInfo) bucketOneObj.getObjectDBInfo()).getName()); - } - - @Test - public void testGetBasicInfoBucketTwo() throws Exception { - // Test bucket 2's basics - Response bucketTwoResponse = - nsSummaryEndpoint.getBasicInfo(BUCKET_TWO_PATH); - NamespaceSummaryResponse bucketTwoObj = - (NamespaceSummaryResponse) bucketTwoResponse.getEntity(); - assertEquals(EntityType.BUCKET, bucketTwoObj.getEntityType()); - assertEquals(2, bucketTwoObj.getCountStats().getNumTotalKey()); - assertEquals(VOL, - ((BucketObjectDBInfo) bucketTwoObj.getObjectDBInfo()).getVolumeName()); - assertEquals(StorageType.DISK, - ((BucketObjectDBInfo) - bucketTwoObj.getObjectDBInfo()).getStorageType()); - assertEquals(getOBSBucketLayout(), - ((BucketObjectDBInfo) - bucketTwoObj.getObjectDBInfo()).getBucketLayout()); - assertEquals(BUCKET_TWO, - ((BucketObjectDBInfo) bucketTwoObj.getObjectDBInfo()).getName()); - } - - @Test - public void testGetBasicInfoBucketThree() throws Exception { - // Test bucket 3's basics - Response bucketThreeResponse = - nsSummaryEndpoint.getBasicInfo(BUCKET_THREE_PATH); - NamespaceSummaryResponse bucketThreeObj = (NamespaceSummaryResponse) - bucketThreeResponse.getEntity(); - assertEquals(EntityType.BUCKET, bucketThreeObj.getEntityType()); - assertEquals(3, bucketThreeObj.getCountStats().getNumTotalKey()); - assertEquals(VOL_TWO, - ((BucketObjectDBInfo) bucketThreeObj.getObjectDBInfo()).getVolumeName()); - assertEquals(StorageType.DISK, - ((BucketObjectDBInfo) - bucketThreeObj.getObjectDBInfo()).getStorageType()); - assertEquals(getLegacyBucketLayout(), - ((BucketObjectDBInfo) - bucketThreeObj.getObjectDBInfo()).getBucketLayout()); - assertEquals(BUCKET_THREE, - ((BucketObjectDBInfo) bucketThreeObj.getObjectDBInfo()).getName()); - } - - @Test - public void testGetBasicInfoBucketFour() throws Exception { - // Test bucket 4's basics - Response bucketFourResponse = - nsSummaryEndpoint.getBasicInfo(BUCKET_FOUR_PATH); - NamespaceSummaryResponse bucketFourObj = - (NamespaceSummaryResponse) bucketFourResponse.getEntity(); - assertEquals(EntityType.BUCKET, bucketFourObj.getEntityType()); - assertEquals(1, bucketFourObj.getCountStats().getNumTotalKey()); - assertEquals(VOL_TWO, - ((BucketObjectDBInfo) bucketFourObj.getObjectDBInfo()).getVolumeName()); - assertEquals(StorageType.DISK, - ((BucketObjectDBInfo) - bucketFourObj.getObjectDBInfo()).getStorageType()); - assertEquals(getLegacyBucketLayout(), - ((BucketObjectDBInfo) - bucketFourObj.getObjectDBInfo()).getBucketLayout()); - assertEquals(BUCKET_FOUR, - ((BucketObjectDBInfo) bucketFourObj.getObjectDBInfo()).getName()); - } - - @Test - public void testGetBasicInfoNoPath() throws Exception { - // Test invalid path - testNSSummaryBasicInfoNoPath(nsSummaryEndpoint); - } - - @Test - public void testGetBasicInfoKey() throws Exception { - // Test key - testNSSummaryBasicInfoKey(nsSummaryEndpoint); - } - - @Test - public void testDiskUsageRoot() throws Exception { - // root level DU - Response rootResponse = nsSummaryEndpoint.getDiskUsage(ROOT_PATH, - false, false, false); - DUResponse duRootRes = (DUResponse) rootResponse.getEntity(); - assertEquals(2, duRootRes.getCount()); - List<DUResponse.DiskUsage> duRootData = duRootRes.getDuData(); - // sort based on subpath - Collections.sort(duRootData, - Comparator.comparing(DUResponse.DiskUsage::getSubpath)); - DUResponse.DiskUsage duVol1 = duRootData.get(0); - DUResponse.DiskUsage duVol2 = duRootData.get(1); - assertEquals(VOL_PATH, duVol1.getSubpath()); - assertEquals(VOL_TWO_PATH, duVol2.getSubpath()); - assertEquals(VOL_DATA_SIZE, duVol1.getSize()); - assertEquals(VOL_TWO_DATA_SIZE, duVol2.getSize()); - } - - @Test - public void testDiskUsageVolume() throws Exception { - // volume level DU - Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, - false, false, false); - DUResponse duVolRes = (DUResponse) volResponse.getEntity(); - assertEquals(2, duVolRes.getCount()); - List<DUResponse.DiskUsage> duData = duVolRes.getDuData(); - // sort based on subpath - Collections.sort(duData, - Comparator.comparing(DUResponse.DiskUsage::getSubpath)); - DUResponse.DiskUsage duBucket1 = duData.get(0); - DUResponse.DiskUsage duBucket2 = duData.get(1); - assertEquals(BUCKET_ONE_PATH, duBucket1.getSubpath()); - assertEquals(BUCKET_TWO_PATH, duBucket2.getSubpath()); - assertEquals(BUCKET_ONE_DATA_SIZE, duBucket1.getSize()); - assertEquals(BUCKET_TWO_DATA_SIZE, duBucket2.getSize()); - } - - @Test - public void testDiskUsageVolTwo() throws Exception { - // volume level DU - Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_TWO_PATH, - false, false, false); - DUResponse duVolRes = (DUResponse) volResponse.getEntity(); - assertEquals(2, duVolRes.getCount()); - List<DUResponse.DiskUsage> duData = duVolRes.getDuData(); - // sort based on subpath - Collections.sort(duData, - Comparator.comparing(DUResponse.DiskUsage::getSubpath)); - DUResponse.DiskUsage duBucket3 = duData.get(0); - DUResponse.DiskUsage duBucket4 = duData.get(1); - assertEquals(BUCKET_THREE_PATH, duBucket3.getSubpath()); - assertEquals(BUCKET_FOUR_PATH, duBucket4.getSubpath()); - assertEquals(VOL_TWO_DATA_SIZE, duVolRes.getSize()); - } - - @Test - public void testDiskUsageBucketOne() throws Exception { - // bucket level DU - Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, - false, false, false); - DUResponse duBucketResponse = (DUResponse) bucketResponse.getEntity(); - // There are no sub-paths under this OBS bucket. - assertEquals(0, duBucketResponse.getCount()); - - Response bucketResponseWithSubpath = nsSummaryEndpoint.getDiskUsage( - BUCKET_ONE_PATH, true, false, false); - DUResponse duBucketResponseWithFiles = - (DUResponse) bucketResponseWithSubpath.getEntity(); - assertEquals(3, duBucketResponseWithFiles.getCount()); - - assertEquals(BUCKET_ONE_DATA_SIZE, duBucketResponse.getSize()); - } - - @Test - public void testDiskUsageBucketTwo() throws Exception { - // bucket level DU - Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_TWO_PATH, - false, false, false); - DUResponse duBucketResponse = (DUResponse) bucketResponse.getEntity(); - // There are no sub-paths under this OBS bucket. - assertEquals(0, duBucketResponse.getCount()); - - Response bucketResponseWithSubpath = nsSummaryEndpoint.getDiskUsage( - BUCKET_TWO_PATH, true, false, false); - DUResponse duBucketResponseWithFiles = - (DUResponse) bucketResponseWithSubpath.getEntity(); - assertEquals(2, duBucketResponseWithFiles.getCount()); - - assertEquals(BUCKET_TWO_DATA_SIZE, duBucketResponse.getSize()); - } - - @Test - public void testDiskUsageBucketThree() throws Exception { - // bucket level DU - Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_THREE_PATH, - false, false, false); - DUResponse duBucketResponse = (DUResponse) bucketResponse.getEntity(); - // There are no sub-paths under this Legacy bucket. - assertEquals(0, duBucketResponse.getCount()); - - Response bucketResponseWithSubpath = nsSummaryEndpoint.getDiskUsage( - BUCKET_THREE_PATH, true, false, false); - DUResponse duBucketResponseWithFiles = - (DUResponse) bucketResponseWithSubpath.getEntity(); - assertEquals(3, duBucketResponseWithFiles.getCount()); - - assertEquals(BUCKET_THREE_DATA_SIZE, duBucketResponse.getSize()); - } - - @Test - public void testDiskUsageKey1() throws Exception { - // key level DU - Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_ONE_PATH, - false, false, false); - DUResponse duKeyResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(0, duKeyResponse.getCount()); - assertEquals(FILE_ONE_SIZE, duKeyResponse.getSize()); - } - - @Test - public void testDiskUsageKey2() throws Exception { - // key level DU - Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_TWO_PATH, - false, false, false); - DUResponse duKeyResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(0, duKeyResponse.getCount()); - assertEquals(FILE_TWO_SIZE, duKeyResponse.getSize()); - } - - @Test - public void testDiskUsageKey4() throws Exception { - // key level DU - Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY4_PATH, - true, false, false); - DUResponse duKeyResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(0, duKeyResponse.getCount()); - assertEquals(FILE_FOUR_SIZE, duKeyResponse.getSize()); - } - - @Test - public void testDiskUsageKey5() throws Exception { - // key level DU - Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_FIVE_PATH, - false, false, false); - DUResponse duKeyResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(0, duKeyResponse.getCount()); - assertEquals(FILE_FIVE_SIZE, duKeyResponse.getSize()); - } - - @Test - public void testDiskUsageKey8() throws Exception { - // key level DU - Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_EIGHT_PATH, - false, false, false); - DUResponse duKeyResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(0, duKeyResponse.getCount()); - assertEquals(FILE_EIGHT_SIZE, duKeyResponse.getSize()); - } - - @Test - public void testDiskUsageKey11() throws Exception { - // key level DU - Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_ELEVEN_PATH, - false, false, false); - DUResponse duKeyResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(0, duKeyResponse.getCount()); - assertEquals(FILE_ELEVEN_SIZE, duKeyResponse.getSize()); - } - - @Test - public void testDiskUsageUnknown() throws Exception { - // invalid path check - Response invalidResponse = nsSummaryEndpoint.getDiskUsage(INVALID_PATH, - false, false, false); - DUResponse invalidObj = (DUResponse) invalidResponse.getEntity(); - assertEquals(ResponseStatus.PATH_NOT_FOUND, - invalidObj.getStatus()); - } - - @Test - public void testDiskUsageWithReplication() throws Exception { - setUpMultiBlockKey(); - Response keyResponse = nsSummaryEndpoint.getDiskUsage(MULTI_BLOCK_KEY_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_KEY_SIZE_WITH_REPLICA, - replicaDUResponse.getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderRootWithReplication() throws IOException { - setUpMultiBlockReplicatedKeys(); - // withReplica is true - Response rootResponse = nsSummaryEndpoint.getDiskUsage(ROOT_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) rootResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_ROOT, - replicaDUResponse.getSizeWithReplica()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_VOL, - replicaDUResponse.getDuData().get(0).getSizeWithReplica()); - - } - - @Test - public void testDataSizeUnderVolWithReplication() throws IOException { - setUpMultiBlockReplicatedKeys(); - Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) volResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_VOL, - replicaDUResponse.getSizeWithReplica()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET1, - replicaDUResponse.getDuData().get(0).getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderBucketOneWithReplication() throws IOException { - setUpMultiBlockReplicatedKeys(); - Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) bucketResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET1, - replicaDUResponse.getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderBucketThreeWithReplication() throws IOException { - setUpMultiBlockReplicatedKeys(); - Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_THREE_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) bucketResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET3, - replicaDUResponse.getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderKeyWithReplication() throws IOException { - setUpMultiBlockReplicatedKeys(); - Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY4_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_KEY, - replicaDUResponse.getSizeWithReplica()); - } - - @Test - public void testQuotaUsage() throws Exception { - // root level quota usage - Response rootResponse = nsSummaryEndpoint.getQuotaUsage(ROOT_PATH); - QuotaUsageResponse quRootRes = - (QuotaUsageResponse) rootResponse.getEntity(); - assertEquals(ROOT_QUOTA, quRootRes.getQuota()); - assertEquals(ROOT_DATA_SIZE, quRootRes.getQuotaUsed()); - - // volume level quota usage - Response volResponse = nsSummaryEndpoint.getQuotaUsage(VOL_PATH); - QuotaUsageResponse quVolRes = (QuotaUsageResponse) volResponse.getEntity(); - assertEquals(VOL_QUOTA, quVolRes.getQuota()); - assertEquals(VOL_DATA_SIZE, quVolRes.getQuotaUsed()); - - // bucket level quota usage - Response bucketRes = nsSummaryEndpoint.getQuotaUsage(BUCKET_ONE_PATH); - QuotaUsageResponse quBucketRes = (QuotaUsageResponse) bucketRes.getEntity(); - assertEquals(BUCKET_ONE_QUOTA, quBucketRes.getQuota()); - assertEquals(BUCKET_ONE_DATA_SIZE, quBucketRes.getQuotaUsed()); - - Response bucketRes2 = nsSummaryEndpoint.getQuotaUsage(BUCKET_TWO_PATH); - QuotaUsageResponse quBucketRes2 = - (QuotaUsageResponse) bucketRes2.getEntity(); - assertEquals(BUCKET_TWO_QUOTA, quBucketRes2.getQuota()); - assertEquals(BUCKET_TWO_DATA_SIZE, quBucketRes2.getQuotaUsed()); - - Response bucketRes3 = nsSummaryEndpoint.getQuotaUsage(BUCKET_THREE_PATH); - QuotaUsageResponse quBucketRes3 = - (QuotaUsageResponse) bucketRes3.getEntity(); - assertEquals(BUCKET_THREE_QUOTA, quBucketRes3.getQuota()); - assertEquals(BUCKET_THREE_DATA_SIZE, quBucketRes3.getQuotaUsed()); - - Response bucketRes4 = nsSummaryEndpoint.getQuotaUsage(BUCKET_FOUR_PATH); - QuotaUsageResponse quBucketRes4 = - (QuotaUsageResponse) bucketRes4.getEntity(); - assertEquals(BUCKET_FOUR_QUOTA, quBucketRes4.getQuota()); - assertEquals(BUCKET_FOUR_DATA_SIZE, quBucketRes4.getQuotaUsed()); - - // other level not applicable - Response naResponse2 = nsSummaryEndpoint.getQuotaUsage(KEY4_PATH); - QuotaUsageResponse quotaUsageResponse2 = - (QuotaUsageResponse) naResponse2.getEntity(); - assertEquals(ResponseStatus.TYPE_NOT_APPLICABLE, - quotaUsageResponse2.getResponseCode()); - - // invalid path request - Response invalidRes = nsSummaryEndpoint.getQuotaUsage(INVALID_PATH); - QuotaUsageResponse invalidResObj = - (QuotaUsageResponse) invalidRes.getEntity(); - assertEquals(ResponseStatus.PATH_NOT_FOUND, - invalidResObj.getResponseCode()); - } - - @Test - public void testFileSizeDist() throws Exception { - checkFileSizeDist(ROOT_PATH, 2, 3, 3, 1); - checkFileSizeDist(VOL_PATH, 2, 1, 1, 1); - checkFileSizeDist(BUCKET_ONE_PATH, 1, 1, 0, 1); - } - - public void checkFileSizeDist(String path, int bin0, - int bin1, int bin2, int bin3) throws Exception { - Response res = nsSummaryEndpoint.getFileSizeDistribution(path); - FileSizeDistributionResponse fileSizeDistResObj = - (FileSizeDistributionResponse) res.getEntity(); - int[] fileSizeDist = fileSizeDistResObj.getFileSizeDist(); - assertEquals(bin0, fileSizeDist[0]); - assertEquals(bin1, fileSizeDist[1]); - assertEquals(bin2, fileSizeDist[2]); - assertEquals(bin3, fileSizeDist[3]); - for (int i = 4; i < ReconConstants.NUM_OF_FILE_SIZE_BINS; ++i) { - assertEquals(0, fileSizeDist[i]); - } - } - - @Test - public void testNormalizePathUptoBucket() { - // Test null or empty path - assertEquals("/", OmUtils.normalizePathUptoBucket(null)); - assertEquals("/", OmUtils.normalizePathUptoBucket("")); - - // Test path with leading slashes - assertEquals("volume1/bucket1/key1/key2", - OmUtils.normalizePathUptoBucket("///volume1/bucket1/key1/key2")); - - // Test volume and bucket names - assertEquals("volume1/bucket1", - OmUtils.normalizePathUptoBucket("volume1/bucket1")); - - // Test with additional segments - assertEquals("volume1/bucket1/key1/key2", - OmUtils.normalizePathUptoBucket("volume1/bucket1/key1/key2")); - - // Test path with multiple slashes in key names. - assertEquals("volume1/bucket1/key1//key2", - OmUtils.normalizePathUptoBucket("volume1/bucket1/key1//key2")); - - // Test path with volume, bucket, and special characters in keys - assertEquals("volume/bucket/key$%#1/./////////key$%#2", - OmUtils.normalizePathUptoBucket("volume/bucket/key$%#1/./////////key$%#2")); - } - - @Test - public void testConstructFullPath() throws IOException { - OmKeyInfo keyInfo = new OmKeyInfo.Builder() - .setKeyName(KEY_TWO) - .setVolumeName(VOL) - .setBucketName(BUCKET_ONE) - .setObjectID(KEY_TWO_OBJECT_ID) - .build(); - String fullPath = ReconUtils.constructFullPath(keyInfo, - reconNamespaceSummaryManager); - String expectedPath = "vol/bucket1/" + KEY_TWO; - Assertions.assertEquals(expectedPath, fullPath); - - keyInfo = new OmKeyInfo.Builder() - .setKeyName(KEY_FIVE) - .setVolumeName(VOL) - .setBucketName(BUCKET_TWO) - .setObjectID(KEY_FIVE_OBJECT_ID) - .build(); - fullPath = ReconUtils.constructFullPath(keyInfo, - reconNamespaceSummaryManager); - expectedPath = "vol/bucket2/" + KEY_FIVE; - Assertions.assertEquals(expectedPath, fullPath); - - keyInfo = new OmKeyInfo.Builder() - .setKeyName(KEY_EIGHT) - .setVolumeName(VOL_TWO) - .setBucketName(BUCKET_THREE) - .setObjectID(KEY_EIGHT_OBJECT_ID) - .build(); - fullPath = ReconUtils.constructFullPath(keyInfo, - reconNamespaceSummaryManager); - expectedPath = "vol2/bucket3/" + KEY_EIGHT; - Assertions.assertEquals(expectedPath, fullPath); - - - keyInfo = new OmKeyInfo.Builder() - .setKeyName(KEY_ELEVEN) - .setVolumeName(VOL_TWO) - .setBucketName(BUCKET_FOUR) - .setObjectID(KEY_ELEVEN_OBJECT_ID) - .build(); - fullPath = ReconUtils.constructFullPath(keyInfo, - reconNamespaceSummaryManager); - expectedPath = "vol2/bucket4/" + KEY_ELEVEN; - Assertions.assertEquals(expectedPath, fullPath); - } - - - /** - * Testing the following case. - * └── vol - * ├── bucket1 (OBS) - * │ ├── file1 - * │ ├── file2 - * │ └── file3 - * └── bucket2 (OBS) - * ├── file4 - * └── file5 - * └── vol2 - * ├── bucket3 (Legacy) - * │ ├── file8 - * │ ├── file9 - * │ └── file10 - * └── bucket4 (Legacy) - * └── file11 - * - * Write these keys to OM and - * replicate them. - * @throws Exception - */ - @SuppressWarnings("checkstyle:MethodLength") - private void populateOMDB() throws Exception { - - // write all keys - writeKeyToOm(reconOMMetadataManager, - KEY_ONE, - BUCKET_ONE, - VOL, - KEY_ONE, - KEY_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - FILE_ONE_SIZE, - getOBSBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_TWO, - BUCKET_ONE, - VOL, - KEY_TWO, - KEY_TWO_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - FILE_TWO_SIZE, - getOBSBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_THREE, - BUCKET_ONE, - VOL, - KEY_THREE, - KEY_THREE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - FILE_THREE_SIZE, - getOBSBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_FOUR, - BUCKET_TWO, - VOL, - KEY_FOUR, - KEY_FOUR_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - VOL_OBJECT_ID, - FILE_FOUR_SIZE, - getOBSBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_FIVE, - BUCKET_TWO, - VOL, - KEY_FIVE, - KEY_FIVE_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - VOL_OBJECT_ID, - FILE_FIVE_SIZE, - getOBSBucketLayout()); - - writeKeyToOm(reconOMMetadataManager, - KEY_EIGHT, - BUCKET_THREE, - VOL_TWO, - KEY_EIGHT, - KEY_EIGHT_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - FILE_EIGHT_SIZE, - getLegacyBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_NINE, - BUCKET_THREE, - VOL_TWO, - KEY_NINE, - KEY_NINE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - FILE_NINE_SIZE, - getLegacyBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_TEN, - BUCKET_THREE, - VOL_TWO, - KEY_TEN, - KEY_TEN_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - FILE_TEN_SIZE, - getLegacyBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_ELEVEN, - BUCKET_FOUR, - VOL_TWO, - KEY_ELEVEN, - KEY_ELEVEN_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_FOUR_OBJECT_ID, - VOL_TWO_OBJECT_ID, - FILE_ELEVEN_SIZE, - getLegacyBucketLayout()); - } - - /** - * Create a new OM Metadata manager instance with one user, one vol, and two - * buckets. - * - * @throws IOException ioEx - */ - private static OMMetadataManager initializeNewOmMetadataManager( - File omDbDir, OzoneConfiguration omConfiguration) - throws IOException { - omConfiguration.set(OZONE_OM_DB_DIRS, - omDbDir.getAbsolutePath()); - omConfiguration.set(OMConfigKeys - .OZONE_OM_ENABLE_FILESYSTEM_PATHS, "false"); - OMMetadataManager omMetadataManager = new OmMetadataManagerImpl( - omConfiguration, null); - - String volumeKey = omMetadataManager.getVolumeKey(VOL); - OmVolumeArgs args = - OmVolumeArgs.newBuilder() - .setObjectID(VOL_OBJECT_ID) - .setVolume(VOL) - .setAdminName(TEST_USER) - .setOwnerName(TEST_USER) - .setQuotaInBytes(VOL_QUOTA) - .build(); - - String volume2Key = omMetadataManager.getVolumeKey(VOL_TWO); - OmVolumeArgs args2 = - OmVolumeArgs.newBuilder() - .setObjectID(VOL_TWO_OBJECT_ID) - .setVolume(VOL_TWO) - .setAdminName(TEST_USER) - .setOwnerName(TEST_USER) - .setQuotaInBytes(VOL_TWO_QUOTA) - .build(); - - omMetadataManager.getVolumeTable().put(volumeKey, args); - omMetadataManager.getVolumeTable().put(volume2Key, args2); - - OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() - .setVolumeName(VOL) - .setBucketName(BUCKET_ONE) - .setObjectID(BUCKET_ONE_OBJECT_ID) - .setQuotaInBytes(BUCKET_ONE_QUOTA) - .setBucketLayout(getOBSBucketLayout()) - .build(); - - OmBucketInfo bucketInfo2 = OmBucketInfo.newBuilder() - .setVolumeName(VOL) - .setBucketName(BUCKET_TWO) - .setObjectID(BUCKET_TWO_OBJECT_ID) - .setQuotaInBytes(BUCKET_TWO_QUOTA) - .setBucketLayout(getOBSBucketLayout()) - .build(); - - OmBucketInfo bucketInfo3 = OmBucketInfo.newBuilder() - .setVolumeName(VOL_TWO) - .setBucketName(BUCKET_THREE) - .setObjectID(BUCKET_THREE_OBJECT_ID) - .setQuotaInBytes(BUCKET_THREE_QUOTA) - .setBucketLayout(getLegacyBucketLayout()) - .build(); - - OmBucketInfo bucketInfo4 = OmBucketInfo.newBuilder() - .setVolumeName(VOL_TWO) - .setBucketName(BUCKET_FOUR) - .setObjectID(BUCKET_FOUR_OBJECT_ID) - .setQuotaInBytes(BUCKET_FOUR_QUOTA) - .setBucketLayout(getLegacyBucketLayout()) - .build(); - - String bucketKey = omMetadataManager.getBucketKey( - bucketInfo.getVolumeName(), bucketInfo.getBucketName()); - String bucketKey2 = omMetadataManager.getBucketKey( - bucketInfo2.getVolumeName(), bucketInfo2.getBucketName()); - String bucketKey3 = omMetadataManager.getBucketKey( - bucketInfo3.getVolumeName(), bucketInfo3.getBucketName()); - String bucketKey4 = omMetadataManager.getBucketKey( - bucketInfo4.getVolumeName(), bucketInfo4.getBucketName()); - - omMetadataManager.getBucketTable().put(bucketKey, bucketInfo); - omMetadataManager.getBucketTable().put(bucketKey2, bucketInfo2); - omMetadataManager.getBucketTable().put(bucketKey3, bucketInfo3); - omMetadataManager.getBucketTable().put(bucketKey4, bucketInfo4); - - return omMetadataManager; - } - - private void setUpMultiBlockKey() throws IOException { - OmKeyLocationInfoGroup locationInfoGroup = - getLocationInfoGroup1(); - - // add the multi-block key to Recon's OM - writeKeyToOm(reconOMMetadataManager, - MULTI_BLOCK_FILE, - BUCKET_ONE, - VOL, - MULTI_BLOCK_FILE, - MULTI_BLOCK_KEY_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup), - getOBSBucketLayout(), - FILE_THREE_SIZE); - } - - private OmKeyLocationInfoGroup getLocationInfoGroup1() { - List<OmKeyLocationInfo> locationInfoList = new ArrayList<>(); - BlockID block1 = new BlockID(CONTAINER_ONE_ID, 0L); - BlockID block2 = new BlockID(CONTAINER_TWO_ID, 0L); - BlockID block3 = new BlockID(CONTAINER_THREE_ID, 0L); - - OmKeyLocationInfo location1 = new OmKeyLocationInfo.Builder() - .setBlockID(block1) - .setLength(BLOCK_ONE_LENGTH) - .build(); - OmKeyLocationInfo location2 = new OmKeyLocationInfo.Builder() - .setBlockID(block2) - .setLength(BLOCK_TWO_LENGTH) - .build(); - OmKeyLocationInfo location3 = new OmKeyLocationInfo.Builder() - .setBlockID(block3) - .setLength(BLOCK_THREE_LENGTH) - .build(); - locationInfoList.add(location1); - locationInfoList.add(location2); - locationInfoList.add(location3); - - return new OmKeyLocationInfoGroup(0L, locationInfoList); - } - - private OmKeyLocationInfoGroup getLocationInfoGroup2() { - List<OmKeyLocationInfo> locationInfoList = new ArrayList<>(); - BlockID block4 = new BlockID(CONTAINER_FOUR_ID, 0L); - BlockID block5 = new BlockID(CONTAINER_FIVE_ID, 0L); - BlockID block6 = new BlockID(CONTAINER_SIX_ID, 0L); - - OmKeyLocationInfo location4 = new OmKeyLocationInfo.Builder() - .setBlockID(block4) - .setLength(BLOCK_FOUR_LENGTH) - .build(); - OmKeyLocationInfo location5 = new OmKeyLocationInfo.Builder() - .setBlockID(block5) - .setLength(BLOCK_FIVE_LENGTH) - .build(); - OmKeyLocationInfo location6 = new OmKeyLocationInfo.Builder() - .setBlockID(block6) - .setLength(BLOCK_SIX_LENGTH) - .build(); - locationInfoList.add(location4); - locationInfoList.add(location5); - locationInfoList.add(location6); - return new OmKeyLocationInfoGroup(0L, locationInfoList); - - } - - @SuppressWarnings("checkstyle:MethodLength") - private void setUpMultiBlockReplicatedKeys() throws IOException { - OmKeyLocationInfoGroup locationInfoGroup1 = - getLocationInfoGroup1(); - OmKeyLocationInfoGroup locationInfoGroup2 = - getLocationInfoGroup2(); - - //vol/bucket1/file1 - writeKeyToOm(reconOMMetadataManager, - KEY_ONE, - BUCKET_ONE, - VOL, - KEY_ONE, - KEY_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getOBSBucketLayout(), - FILE_ONE_SIZE); - - //vol/bucket1/file2 - writeKeyToOm(reconOMMetadataManager, - KEY_TWO, - BUCKET_ONE, - VOL, - KEY_TWO, - KEY_TWO_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getOBSBucketLayout(), - FILE_TWO_SIZE); - - //vol/bucket1/file3 - writeKeyToOm(reconOMMetadataManager, - KEY_THREE, - BUCKET_ONE, - VOL, - KEY_THREE, - KEY_THREE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getOBSBucketLayout(), - FILE_THREE_SIZE); - - //vol/bucket2/file4 - writeKeyToOm(reconOMMetadataManager, - KEY_FOUR, - BUCKET_TWO, - VOL, - KEY_FOUR, - KEY_FOUR_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getOBSBucketLayout(), - FILE_FOUR_SIZE); - - //vol/bucket2/file5 - writeKeyToOm(reconOMMetadataManager, - KEY_FIVE, - BUCKET_TWO, - VOL, - KEY_FIVE, - KEY_FIVE_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getOBSBucketLayout(), - FILE_FIVE_SIZE); - - //vol2/bucket3/file8 - writeKeyToOm(reconOMMetadataManager, - KEY_EIGHT, - BUCKET_THREE, - VOL_TWO, - KEY_EIGHT, - KEY_EIGHT_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getLegacyBucketLayout(), - FILE_EIGHT_SIZE); - - //vol2/bucket3/file9 - writeKeyToOm(reconOMMetadataManager, - KEY_NINE, - BUCKET_THREE, - VOL_TWO, - KEY_NINE, - KEY_NINE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getLegacyBucketLayout(), - FILE_NINE_SIZE); - - //vol2/bucket3/file10 - writeKeyToOm(reconOMMetadataManager, - KEY_TEN, - BUCKET_THREE, - VOL_TWO, - KEY_TEN, - KEY_TEN_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getLegacyBucketLayout(), - FILE_TEN_SIZE); - - //vol2/bucket4/file11 - writeKeyToOm(reconOMMetadataManager, - KEY_ELEVEN, - BUCKET_FOUR, - VOL_TWO, - KEY_ELEVEN, - KEY_ELEVEN_OBJECT_ID, - BUCKET_FOUR_OBJECT_ID, - BUCKET_FOUR_OBJECT_ID, - VOL_TWO_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getLegacyBucketLayout(), - FILE_ELEVEN_SIZE); - } - - /** - * Generate a set of mock container replica with a size of - * replication factor for container. - * - * @param replicationFactor number of replica - * @param containerID the container replicated based upon - * @return a set of container replica for testing - */ - private static Set<ContainerReplica> generateMockContainerReplicas( - int replicationFactor, ContainerID containerID) { - Set<ContainerReplica> result = new HashSet<>(); - for (int i = 0; i < replicationFactor; ++i) { - DatanodeDetails randomDatanode = randomDatanodeDetails(); - ContainerReplica replica = new ContainerReplica.ContainerReplicaBuilder() - .setContainerID(containerID) - .setContainerState( - StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.OPEN) - .setDatanodeDetails(randomDatanode) - .build(); - result.add(replica); - } - return result; - } - - private static ReconStorageContainerManagerFacade getMockReconSCM() - throws ContainerNotFoundException { - ReconStorageContainerManagerFacade reconSCM = - mock(ReconStorageContainerManagerFacade.class); - ContainerManager containerManager = mock(ContainerManager.class); - - // Container 1 is 3-way replicated - ContainerID containerID1 = ContainerID.valueOf(CONTAINER_ONE_ID); - Set<ContainerReplica> containerReplicas1 = generateMockContainerReplicas( - CONTAINER_ONE_REPLICA_COUNT, containerID1); - when(containerManager.getContainerReplicas(containerID1)) - .thenReturn(containerReplicas1); - - // Container 2 is under replicated with 2 replica - ContainerID containerID2 = ContainerID.valueOf(CONTAINER_TWO_ID); - Set<ContainerReplica> containerReplicas2 = generateMockContainerReplicas( - CONTAINER_TWO_REPLICA_COUNT, containerID2); - when(containerManager.getContainerReplicas(containerID2)) - .thenReturn(containerReplicas2); - - // Container 3 is over replicated with 4 replica - ContainerID containerID3 = ContainerID.valueOf(CONTAINER_THREE_ID); - Set<ContainerReplica> containerReplicas3 = generateMockContainerReplicas( - CONTAINER_THREE_REPLICA_COUNT, containerID3); - when(containerManager.getContainerReplicas(containerID3)) - .thenReturn(containerReplicas3); - - // Container 4 is replicated with 5 replica - ContainerID containerID4 = ContainerID.valueOf(CONTAINER_FOUR_ID); - Set<ContainerReplica> containerReplicas4 = generateMockContainerReplicas( - CONTAINER_FOUR_REPLICA_COUNT, containerID4); - when(containerManager.getContainerReplicas(containerID4)) - .thenReturn(containerReplicas4); - - // Container 5 is replicated with 2 replica - ContainerID containerID5 = ContainerID.valueOf(CONTAINER_FIVE_ID); - Set<ContainerReplica> containerReplicas5 = generateMockContainerReplicas( - CONTAINER_FIVE_REPLICA_COUNT, containerID5); - when(containerManager.getContainerReplicas(containerID5)) - .thenReturn(containerReplicas5); - - // Container 6 is replicated with 3 replica - ContainerID containerID6 = ContainerID.valueOf(CONTAINER_SIX_ID); - Set<ContainerReplica> containerReplicas6 = generateMockContainerReplicas( - CONTAINER_SIX_REPLICA_COUNT, containerID6); - when(containerManager.getContainerReplicas(containerID6)) - .thenReturn(containerReplicas6); - - when(reconSCM.getContainerManager()).thenReturn(containerManager); - ReconNodeManager mockReconNodeManager = mock(ReconNodeManager.class); - when(mockReconNodeManager.getStats()).thenReturn(getMockSCMRootStat()); - when(reconSCM.getScmNodeManager()).thenReturn(mockReconNodeManager); - return reconSCM; - } - - private static BucketLayout getOBSBucketLayout() { - return BucketLayout.OBJECT_STORE; - } - - private static BucketLayout getLegacyBucketLayout() { - return BucketLayout.LEGACY; - } - - private static SCMNodeStat getMockSCMRootStat() { - return new SCMNodeStat(ROOT_QUOTA, ROOT_DATA_SIZE, - ROOT_QUOTA - ROOT_DATA_SIZE, 0L, 0L, 0); - } - -}
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestOzoneManagerServiceProviderImpl.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestOzoneManagerServiceProviderImpl.java index c566493..de28008 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestOzoneManagerServiceProviderImpl.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestOzoneManagerServiceProviderImpl.java
@@ -39,6 +39,7 @@ import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -64,6 +65,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.recon.ReconContext; import org.apache.hadoop.ozone.recon.ReconUtils; +import org.apache.hadoop.ozone.recon.api.types.OMDBReprocessResponse; import org.apache.hadoop.ozone.recon.common.ReconTestUtils; import org.apache.hadoop.ozone.recon.metrics.OzoneManagerSyncMetrics; import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; @@ -677,6 +679,44 @@ private ReconTaskStatusUpdaterManager getMockTaskStatusUpdaterManager() { return reconTaskStatusUpdaterManager; } + @Test + public void testTriggerTaskRebuild() throws Exception { + ReconTaskController reconTaskController = mock(ReconTaskController.class); + ReconTaskStatusUpdaterManager taskStatusUpdaterManager = mock(ReconTaskStatusUpdaterManager.class); + ReconUtils reconUtils = new ReconUtils(); + + // Test 1: Successful queue (OM DB is loaded) + ReconOMMetadataManager loadedOmMgr = mock(ReconOMMetadataManager.class); + when(loadedOmMgr.getStore()).thenReturn(mock(RDBStore.class)); + OzoneManagerServiceProviderImpl serviceProvider = new OzoneManagerServiceProviderImpl( + configuration, loadedOmMgr, reconTaskController, + reconUtils, ozoneManagerProtocol, reconContext, taskStatusUpdaterManager); + + when(reconTaskController.queueReInitializationEvent( + ReconTaskReInitializationEvent.ReInitializationReason.MANUAL_OM_DB_REBUILD)) + .thenReturn(ReconTaskController.ReInitializationResult.SUCCESS); + + OMDBReprocessResponse response = serviceProvider.triggerTaskRebuild(); + assertEquals(OMDBReprocessResponse.Status.ACCEPTED, response.getStatus()); + + // Test 2: Buffer full / retry + when(reconTaskController.queueReInitializationEvent( + ReconTaskReInitializationEvent.ReInitializationReason.MANUAL_OM_DB_REBUILD)) + .thenReturn(ReconTaskController.ReInitializationResult.RETRY_LATER); + + response = serviceProvider.triggerTaskRebuild(); + assertEquals(OMDBReprocessResponse.Status.RETRY, response.getStatus()); + + // Test 3: No OM DB loaded (store is null) -> RETRY, nothing queued + ReconTaskController noDbController = mock(ReconTaskController.class); + serviceProvider = new OzoneManagerServiceProviderImpl( + configuration, mock(ReconOMMetadataManager.class), noDbController, + reconUtils, ozoneManagerProtocol, reconContext, taskStatusUpdaterManager); + response = serviceProvider.triggerTaskRebuild(); + assertEquals(OMDBReprocessResponse.Status.RETRY, response.getStatus()); + verify(noDbController, never()).queueReInitializationEvent(any()); + } + private BucketLayout getBucketLayout() { return BucketLayout.DEFAULT; }
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconContainerMetadataManagerImpl.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconContainerMetadataManagerImpl.java index 22c3d6d..effe6ba 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconContainerMetadataManagerImpl.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconContainerMetadataManagerImpl.java
@@ -26,6 +26,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; +import java.lang.reflect.Field; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; @@ -156,6 +157,27 @@ public void testInitNewContainerDB() throws Exception { } @Test + public void testReinitWithUninitializedTables() throws Exception { + ReconContainerMetadataManagerImpl manager = + (ReconContainerMetadataManagerImpl) reconContainerMetadataManager; + setFieldToNull(manager, "containerKeyTable"); + setFieldToNull(manager, "keyContainerTable"); + setFieldToNull(manager, "containerKeyCountTable"); + + manager.reinitWithNewContainerDataFromOm(null); + + assertNotNull(manager.getContainerKeyTableForTesting()); + assertNotNull(manager.getKeyContainerTable()); + assertEquals(0, manager.getKeyCountForContainer(1L)); + } + + private static void setFieldToNull(ReconContainerMetadataManagerImpl manager, String fieldName) throws Exception { + Field field = ReconContainerMetadataManagerImpl.class.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(manager, null); + } + + @Test public void testBatchStoreContainerKeyMapping() throws Exception { long containerId = System.currentTimeMillis();
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java index d4b6bbe..eb77f72 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java
@@ -26,6 +26,7 @@ import static org.apache.hadoop.ozone.recon.tasks.OMDBUpdateEvent.OMDBUpdateAction.UPDATE; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.AdditionalAnswers.returnsElementsOf; import static org.mockito.BDDMockito.given; @@ -42,7 +43,6 @@ import java.util.List; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.hdds.utils.db.TypedTable; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; @@ -93,16 +93,17 @@ public void setUp() throws IOException { // Create separate task instances. fileSizeCountTaskOBS = new FileSizeCountTaskOBS(reconFileMetadataManager, configuration); fileSizeCountTaskFSO = new FileSizeCountTaskFSO(reconFileMetadataManager, configuration); - // Clear RocksDB table before each test. - try (TableIterator<FileSizeCountKey, Table.KeyValue<FileSizeCountKey, Long>> iterator = - reconFileMetadataManager.getFileCountTable().iterator()) { - while (iterator.hasNext()) { - Table.KeyValue<FileSizeCountKey, Long> keyValue = iterator.next(); - reconFileMetadataManager.getFileCountTable().delete(keyValue.getKey()); - } - } catch (Exception e) { - // Ignore cleanup errors - } + reconFileMetadataManager.clearFileCountTable(); + } + + @Test + public void testClearFileCountTable() throws IOException { + FileSizeCountKey key = new FileSizeCountKey("vol1", "bucket1", 1024L); + reconFileMetadataManager.getFileCountTable().put(key, 1L); + + reconFileMetadataManager.clearFileCountTable(); + + assertNull(reconFileMetadataManager.getFileSizeCount(key)); } @Test
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOmTableInsightTask.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOmTableInsightTask.java index 92ee526..ef44075 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOmTableInsightTask.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOmTableInsightTask.java
@@ -58,12 +58,16 @@ import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TypedTable; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmMultipartPartInfo; +import org.apache.hadoop.ozone.om.helpers.OmMultipartPartKey; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo; @@ -783,6 +787,131 @@ public void testReprocessForMultipartInfoTable() throws Exception { assertEquals(900L, getReplicatedSizeForTable(MULTIPART_INFO_TABLE)); } + /** + * Validates schema-aware reprocess for a split (schemaVersion 1) MPU: the + * multipartInfoTable value carries NO embedded parts, so Recon must read the + * part sizes from the separate multipartPartsTable, using the parent MPU's + * replication config to compute the replicated size. + */ + @Test + public void testReprocessForMultipartInfoTableSplitSchema() throws Exception { + String uploadID = UUID.randomUUID().toString(); + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + String keyName = UUID.randomUUID().toString(); + + // Split-schema MPU: no embedded parts in the multipartInfoTable value. + OmMultipartKeyInfo splitMpu = new OmMultipartKeyInfo.Builder() + .setObjectID(1L) + .setUploadID(uploadID) + .setCreationTime(Time.now()) + .setReplicationConfig(RatisReplicationConfig.getInstance( + HddsProtos.ReplicationFactor.THREE)) + .setSchemaVersion(OmMultipartKeyInfo.SPLIT_PARTS_TABLE_SCHEMA_VERSION) + .build(); + String multipartKey = reconOMMetadataManager.getMultipartKey(volumeName, bucketName, keyName, uploadID); + reconOMMetadataManager.getMultipartInfoTable().put(multipartKey, splitMpu); + + // 3 parts of 100 bytes each, stored as rows in the split parts table. + putSplitSchemaPart(uploadID, volumeName, bucketName, keyName, 1, 100L); + putSplitSchemaPart(uploadID, volumeName, bucketName, keyName, 2, 100L); + putSplitSchemaPart(uploadID, volumeName, bucketName, keyName, 3, 100L); + + ReconOmTask.TaskResult result = omTableInsightTask.reprocess(reconOMMetadataManager); + assertTrue(result.isTaskSuccess()); + + // Count is the number of MPU uploads (1), not the number of parts. + assertEquals(1L, getCountForTable(MULTIPART_INFO_TABLE)); + // 3 split-table parts * 100 bytes = 300 bytes unreplicated. + assertEquals(300L, getUnReplicatedSizeForTable(MULTIPART_INFO_TABLE)); + // RATIS THREE: 300 bytes * 3 = 900 bytes replicated. + assertEquals(900L, getReplicatedSizeForTable(MULTIPART_INFO_TABLE)); + } + + /** + * Validates schema-aware reprocess when both legacy (schemaVersion 0, embedded + * parts) and split (schemaVersion 1, parts in multipartPartsTable) MPUs coexist + * in the same DB. Recon must sum both under the multipartInfoTable stats. + */ + @Test + public void testReprocessForMultipartInfoTableMixedSchema() throws Exception { + // Legacy MPU: 2 embedded parts of 100 bytes. + String legacyUploadID = UUID.randomUUID().toString(); + String legacyVol = UUID.randomUUID().toString(); + String legacyBucket = UUID.randomUUID().toString(); + String legacyKey = UUID.randomUUID().toString(); + OmMultipartKeyInfo legacyMpu = new OmMultipartKeyInfo.Builder() + .setObjectID(1L) + .setUploadID(legacyUploadID) + .setCreationTime(Time.now()) + .setReplicationConfig(RatisReplicationConfig.getInstance( + HddsProtos.ReplicationFactor.THREE)) + // schemaVersion defaults to LEGACY_SCHEMA_VERSION (0). + .build(); + legacyMpu.addPartKeyInfo(createPartKeyInfo(legacyVol, legacyBucket, legacyKey, legacyUploadID, 1, 100L)); + legacyMpu.addPartKeyInfo(createPartKeyInfo(legacyVol, legacyBucket, legacyKey, legacyUploadID, 2, 100L)); + reconOMMetadataManager.getMultipartInfoTable().put( + reconOMMetadataManager.getMultipartKey(legacyVol, legacyBucket, legacyKey, legacyUploadID), legacyMpu); + + // Split MPU: 3 parts of 100 bytes in the split parts table. + String splitUploadID = UUID.randomUUID().toString(); + String splitVol = UUID.randomUUID().toString(); + String splitBucket = UUID.randomUUID().toString(); + String splitKey = UUID.randomUUID().toString(); + OmMultipartKeyInfo splitMpu = new OmMultipartKeyInfo.Builder() + .setObjectID(2L) + .setUploadID(splitUploadID) + .setCreationTime(Time.now()) + .setReplicationConfig(RatisReplicationConfig.getInstance( + HddsProtos.ReplicationFactor.THREE)) + .setSchemaVersion(OmMultipartKeyInfo.SPLIT_PARTS_TABLE_SCHEMA_VERSION) + .build(); + reconOMMetadataManager.getMultipartInfoTable().put( + reconOMMetadataManager.getMultipartKey(splitVol, splitBucket, splitKey, splitUploadID), splitMpu); + putSplitSchemaPart(splitUploadID, splitVol, splitBucket, splitKey, 1, 100L); + putSplitSchemaPart(splitUploadID, splitVol, splitBucket, splitKey, 2, 100L); + putSplitSchemaPart(splitUploadID, splitVol, splitBucket, splitKey, 3, 100L); + + ReconOmTask.TaskResult result = omTableInsightTask.reprocess(reconOMMetadataManager); + assertTrue(result.isTaskSuccess()); + + // 2 MPU uploads total (1 legacy + 1 split). + assertEquals(2L, getCountForTable(MULTIPART_INFO_TABLE)); + // Legacy 2*100 (embedded) + split 3*100 (parts table) = 500 bytes unreplicated. + assertEquals(500L, getUnReplicatedSizeForTable(MULTIPART_INFO_TABLE)); + // RATIS THREE: 500 bytes * 3 = 1500 bytes replicated. + assertEquals(1500L, getReplicatedSizeForTable(MULTIPART_INFO_TABLE)); + } + + /** + * Writes a single split-schema part (an {@link OmMultipartPartInfo}) to the + * multipartPartsTable, keyed by {@code uploadId/partNumber}. An ETag is set + * because it is mandatory for split-schema parts. + */ + private void putSplitSchemaPart(String uploadID, String volumeName, String bucketName, + String keyName, int partNumber, long dataSize) throws IOException { + OmKeyInfo partOmKeyInfo = new OmKeyInfo.Builder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(keyName) + .setReplicationConfig(RatisReplicationConfig.getInstance( + HddsProtos.ReplicationFactor.THREE)) + .setDataSize(dataSize) + .setCreationTime(Time.now()) + .setModificationTime(Time.now()) + .setObjectID(UUID.randomUUID().hashCode()) + .setUpdateID(1L) + .setOmKeyLocationInfos(Collections.singletonList( + new OmKeyLocationInfoGroup(0L, new ArrayList<>(), true))) + // ETag is mandatory for split-schema parts. + .addMetadata(OzoneConsts.ETAG, "etag-" + partNumber) + .build(); + String partName = reconOMMetadataManager.getMultipartKey(volumeName, bucketName, keyName, uploadID); + OmMultipartPartInfo partInfo = OmMultipartPartInfo.from(partName, partNumber, partOmKeyInfo); + reconOMMetadataManager.getMultipartPartsTable().put( + OmMultipartPartKey.of(uploadID, partNumber), partInfo); + } + public PartKeyInfo createPartKeyInfo(String volumeName, String bucketName, String keyName, String uploadID, int partNumber, long dataSize) { return PartKeyInfo.newBuilder()
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestReconTaskControllerImpl.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestReconTaskControllerImpl.java index 5fefd4e..e9da0f7 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestReconTaskControllerImpl.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestReconTaskControllerImpl.java
@@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyMap; @@ -813,11 +814,22 @@ public void testProcessReInitializationEventWithCheckpointedManager() throws Exc verify(mockCheckpointedManager, times(1)).close(); } - /** - * Helper method for getting a mocked Task. - * @param taskName name of the task. - * @return instance of reconOmTask. - */ + @Test + public void testCreateOMCheckpointThrowsWhenCheckpointNull() throws Exception { + // getStore().getCheckpoint() returns null when RocksDB fails to snapshot an + // incomplete/corrupt on-disk OM DB; createOMCheckpoint must surface this as an + // IOException so the caller handles it gracefully instead of NPE-ing. + ReconOMMetadataManager omMetadataManager = mock(ReconOMMetadataManager.class); + DBStore dbStore = mock(DBStore.class); + when(omMetadataManager.getStore()).thenReturn(dbStore); + File tempDir = Paths.get(System.getProperty("java.io.tmpdir"), "recon-test").toFile(); + when(dbStore.getDbLocation()).thenReturn(tempDir); + when(dbStore.getCheckpoint(anyString(), any(Boolean.class))).thenReturn(null); + + ReconTaskControllerImpl controller = (ReconTaskControllerImpl) reconTaskController; + assertThrows(IOException.class, () -> controller.createOMCheckpoint(omMetadataManager)); + } + private ReconOmTask getMockTask(String taskName) { ReconOmTask reconOmTaskMock = mock(ReconOmTask.class); when(reconOmTaskMock.getTaskName()).thenReturn(taskName);
diff --git a/hadoop-ozone/s3gateway/pom.xml b/hadoop-ozone/s3gateway/pom.xml index 0e68b5c..531138b 100644 --- a/hadoop-ozone/s3gateway/pom.xml +++ b/hadoop-ozone/s3gateway/pom.xml
@@ -177,11 +177,6 @@ </dependency> <dependency> <groupId>org.glassfish.jersey.core</groupId> - <artifactId>jersey-common</artifactId> - <!-- <scope>test</scope> but transitive via jersey-server --> - </dependency> - <dependency> - <groupId>org.glassfish.jersey.core</groupId> <artifactId>jersey-server</artifactId> </dependency> <dependency>
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketCrudHandler.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketCrudHandler.java index 1e0f243..23c4da2 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketCrudHandler.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketCrudHandler.java
@@ -201,11 +201,7 @@ public Response putBucketLifecycleConfiguration(S3RequestContext context, String } catch (WebApplicationException ex) { throw S3ErrorTable.newError(S3ErrorTable.MALFORMED_XML, bucketName); } catch (OMException ex) { - if (ex.getResult() == OMException.ResultCodes.ACCESS_DENIED) { - throw S3ErrorTable.newError(S3ErrorTable.ACCESS_DENIED, bucketName); - } else if (ex.getResult() == OMException.ResultCodes.INVALID_REQUEST) { - throw S3ErrorTable.newError(S3ErrorTable.INVALID_REQUEST, bucketName, ex); - } + throw S3ErrorTable.newError(bucketName, ex); } return Response.ok().build(); }
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index 438ed93..ad54566 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
@@ -272,6 +272,9 @@ public Response put( try { return handler.handlePutRequest(context, bucketName, body); } catch (OMException ex) { + if (ex.getResult() == ResultCodes.BUCKET_ALREADY_EXISTS) { + throw newDuplicateBucketError(bucketName, ex); + } throw newError(bucketName, ex); } }
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index 452303d..3f960cf 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
@@ -28,9 +28,12 @@ import static org.apache.hadoop.ozone.OzoneConsts.KB; import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_CLIENT_BUFFER_SIZE_DEFAULT; import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_CLIENT_BUFFER_SIZE_KEY; +import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.BUCKET_ALREADY_EXISTS; +import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.BUCKET_ALREADY_OWNED_BY_YOU; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_ARGUMENT; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_REQUEST; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_TAG; +import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_URI; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError; import static org.apache.hadoop.ozone.s3.util.S3Consts.AWS_TAG_PREFIX; import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX; @@ -107,6 +110,7 @@ import org.apache.hadoop.ozone.s3.signature.SignatureInfo; import org.apache.hadoop.ozone.s3.util.AuditUtils; import org.apache.hadoop.ozone.s3.util.S3Utils; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.http.NameValuePair; import org.apache.http.client.utils.URLEncodedUtils; import org.apache.ratis.util.function.CheckedRunnable; @@ -244,6 +248,35 @@ protected OzoneVolume getVolume() throws IOException { } /** + * Maps a duplicate bucket create to the S3 error expected by AWS when the + * requester already owns the bucket name. + */ + protected OS3Exception newDuplicateBucketError(String bucketName, OMException cause) { + try { + OzoneBucket existingBucket = getVolume().getBucket(bucketName); + if (isSameBucketOwner(existingBucket.getOwner())) { + return newError(BUCKET_ALREADY_OWNED_BY_YOU, bucketName, cause); + } + } catch (IOException ex) { + LOG.debug("Could not resolve duplicate bucket owner for {}", bucketName, ex); + } + return newError(BUCKET_ALREADY_EXISTS, bucketName, cause); + } + + private boolean isSameBucketOwner(String bucketOwner) { + String requestOwner = getRequestOwner(); + return requestOwner != null && requestOwner.equals(bucketOwner); + } + + private String getRequestOwner() { + if (s3Auth == null || s3Auth.getUserPrincipal() == null) { + return null; + } + return UserGroupInformation.createRemoteUser(s3Auth.getUserPrincipal()) + .getShortUserName(); + } + + /** * Returns Iterator to iterate over all buckets for a specific user. * The result can be restricted using bucket prefix, will return all * buckets if bucket prefix is null. @@ -617,6 +650,22 @@ protected static boolean isAccessDenied(OMException ex) { || result == ResultCodes.INVALID_TOKEN; } + /** + * Reject object keys that cannot be represented in a valid URI. AWS S3 returns + * InvalidURI for keys containing malformed UTF-8 or ISO control characters. + */ + protected void validateObjectKeyUri(String keyPath) throws OS3Exception { + if (keyPath == null || keyPath.indexOf('\uFFFD') >= 0) { + throw newError(INVALID_URI, keyPath); + } + + for (int i = 0; i < keyPath.length(); i++) { + if (Character.isISOControl(keyPath.charAt(i))) { + throw newError(INVALID_URI, keyPath); + } + } + } + protected ReplicationConfig getReplicationConfig(OzoneBucket ozoneBucket) throws OS3Exception { String storageType = getHeaders().getHeaderString(STORAGE_CLASS_HEADER); String storageConfig = getHeaders().getHeaderString(CUSTOM_METADATA_HEADER_PREFIX + STORAGE_CONFIG_HEADER);
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index 3989fee..ed9602d 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -364,6 +364,7 @@ public Response get( ) throws IOException, OS3Exception { ObjectRequestContext context = new ObjectRequestContext(S3GAction.GET_KEY, bucketName); try { + validateObjectKeyUri(keyPath); return handler.handleGetRequest(context, keyPath); } catch (OMException ex) { throw newError(bucketName, keyPath, ex); @@ -544,6 +545,28 @@ static void addTagCountIfAny( } } + /** + * Guarantees that an S3-originated multipart upload part carries an ETag + * before it is committed. S3 requires every uploaded part to have an ETag + * (see UploadPart and CompleteMultipartUpload), and all S3 gateway + * part-write paths stamp the MD5 of the part as its ETag. This pre-commit + * check enforces that invariant so no S3 part can be committed without one + * (e.g. an UploadPartCopy whose source key has no ETag). + * + * <p>Note: the Ozone Manager also enforces a mandatory ETag server-side for + * every committed part (in the split parts-table schema) for ALL clients, + * not just S3. This gateway-side check is an earlier, S3-native failure so + * the client gets a clean S3 error instead of an OM INVALID_REQUEST. + */ + private static void requirePartETag(Map<String, String> metadata) + throws IOException { + if (metadata == null + || StringUtils.isBlank(metadata.get(OzoneConsts.ETAG))) { + throw new IOException( + "S3 multipart upload part cannot be committed without an ETag"); + } + } + static void addEntityTagHeader(ResponseBuilder responseBuilder, OzoneKey key) { String eTag = key.getMetadata().get(OzoneConsts.ETAG); if (eTag != null) { @@ -963,6 +986,8 @@ private Response createMultipartKey(OzoneVolume volume, OzoneBucket ozoneBucket, if (raw != null) { writeGuard.getMetadata().put(OzoneConsts.ETAG, stripQuotes(raw)); } + writeGuard.addPreCommit( + () -> requirePartETag(writeGuard.getMetadata())); outputStream = ozoneOutputStream; } getMetrics().incCopyObjectSuccessLength(copyLength); @@ -987,6 +1012,8 @@ private Response createMultipartKey(OzoneVolume volume, OzoneBucket ozoneBucket, writeGuard.addPreCommit(checkContentMD5Hook); } writeGuard.getMetadata().put(OzoneConsts.ETAG, md5Hash); + writeGuard.addPreCommit( + () -> requirePartETag(writeGuard.getMetadata())); outputStream = ozoneOutputStream; } getMetrics().incPutKeySuccessLength(putLength);
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3LifecycleConfiguration.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3LifecycleConfiguration.java index ff72a32..de1de96 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3LifecycleConfiguration.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3LifecycleConfiguration.java
@@ -479,7 +479,7 @@ private static Expiration convertFromOzoneExpiration( if (date != null && !date.isEmpty()) { expiration.setDate(date); } - if (ozoneExpiration.getDays() > 0) { + if (ozoneExpiration.getDays() != null && ozoneExpiration.getDays() > 0) { expiration.setDays(ozoneExpiration.getDays()); } @@ -497,7 +497,8 @@ private static AbortIncompleteMultipartUpload convertFromOzoneAbortIncompleteMul AbortIncompleteMultipartUpload abortIncompleteMultipartUpload = new AbortIncompleteMultipartUpload(); - if (ozoneAbortIncompleteMultipartUpload.getDaysAfterInitiation() > 0) { + if (ozoneAbortIncompleteMultipartUpload.getDaysAfterInitiation() != null + && ozoneAbortIncompleteMultipartUpload.getDaysAfterInitiation() > 0) { abortIncompleteMultipartUpload.setDaysAfterInitiation( ozoneAbortIncompleteMultipartUpload.getDaysAfterInitiation()); }
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java index b8240ba..d59aa36 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
@@ -134,6 +134,10 @@ public enum S3ErrorTable { "BucketAlreadyExists", "The requested bucket name is not available" + " as it already exists.", HTTP_CONFLICT), + BUCKET_ALREADY_OWNED_BY_YOU( + "BucketAlreadyOwnedByYou", "Your previous request to create the named" + + " bucket succeeded and you already own it.", HTTP_CONFLICT), + INVALID_TAG( "InvalidTag", "Your request contains tag input that is not valid.", HTTP_BAD_REQUEST),
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java index cb01d28..414c397 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java
@@ -124,6 +124,7 @@ public void createBucket(String bucketName, BucketArgs bucketArgs) throws OMExce buckets.put(bucketName, OzoneBucketStub.newBuilder() .setVolumeName(getName()) .setName(bucketName) + .setOwner(bucketArgs.getOwner()) .setDefaultReplicationConfig(new DefaultReplicationConfig( RatisReplicationConfig.getInstance( HddsProtos.ReplicationFactor.THREE)))
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestVirtualHostStyleFilter.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestVirtualHostStyleFilter.java index d6072e2..6120c34 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestVirtualHostStyleFilter.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestVirtualHostStyleFilter.java
@@ -18,21 +18,28 @@ package org.apache.hadoop.ozone.s3; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.net.URI; +import javax.ws.rs.container.ContainerRequestContext; import javax.ws.rs.core.HttpHeaders; -import javax.ws.rs.core.SecurityContext; +import javax.ws.rs.core.MultivaluedHashMap; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.UriBuilder; +import javax.ws.rs.core.UriInfo; import org.apache.hadoop.fs.InvalidRequestException; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.glassfish.jersey.internal.PropertiesDelegate; -import org.glassfish.jersey.server.ContainerRequest; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import org.mockito.ArgumentCaptor; /** * This class test virtual host style mapping conversion to path style. @@ -51,47 +58,31 @@ public void setup() { conf.set(S3GatewayConfigKeys.OZONE_S3G_DOMAIN_NAME, s3HttpAddr); } - /** - * Create containerRequest object. - * @return ContainerRequest - * @throws Exception - */ - public ContainerRequest createContainerRequest(String host, String path, - String queryParams, - boolean virtualHostStyle) - throws Exception { - URI baseUri = new URI("http://" + s3HttpAddr); - URI virtualHostStyleUri; - if (path == null && queryParams == null) { - virtualHostStyleUri = new URI("http://" + s3HttpAddr); - } else if (path != null && queryParams == null) { - virtualHostStyleUri = new URI("http://" + s3HttpAddr + path); - } else if (path != null && queryParams != null) { - virtualHostStyleUri = new URI("http://" + s3HttpAddr + path + - queryParams); - } else { - virtualHostStyleUri = new URI("http://" + s3HttpAddr + queryParams); + private ContainerRequestContext createRequestContext(String host, + String path) { + return createRequestContext(host, path, new MultivaluedHashMap<>()); + } + + private ContainerRequestContext createRequestContext(String host, String path, + MultivaluedMap<String, String> queryParams) { + URI baseUri = URI.create("http://" + s3HttpAddr); + UriBuilder requestUriBuilder = UriBuilder.fromUri(baseUri); + if (path != null) { + requestUriBuilder.path(path); } - URI pathStyleUri; - if (queryParams == null) { - pathStyleUri = new URI("http://" + s3HttpAddr + path); - } else { - pathStyleUri = new URI("http://" + s3HttpAddr + path + queryParams); - } - String httpMethod = "DELETE"; - SecurityContext securityContext = mock(SecurityContext.class); - PropertiesDelegate propertiesDelegate = mock(PropertiesDelegate.class); - ContainerRequest containerRequest; - if (virtualHostStyle) { - containerRequest = new ContainerRequest(baseUri, virtualHostStyleUri, - httpMethod, securityContext, propertiesDelegate); - containerRequest.header(HttpHeaders.HOST, host); - } else { - containerRequest = new ContainerRequest(baseUri, pathStyleUri, - httpMethod, securityContext, propertiesDelegate); - containerRequest.header(HttpHeaders.HOST, host); - } - return containerRequest; + queryParams.forEach((key, values) -> + requestUriBuilder.queryParam(key, values.toArray())); + + UriInfo uriInfo = mock(UriInfo.class); + when(uriInfo.getBaseUri()).thenReturn(baseUri); + when(uriInfo.getPath()).thenReturn(path == null ? "" : path.substring(1)); + when(uriInfo.getQueryParameters()).thenReturn(queryParams); + when(uriInfo.getRequestUri()).thenReturn(requestUriBuilder.build()); + + ContainerRequestContext requestContext = mock(ContainerRequestContext.class); + when(requestContext.getHeaderString(HttpHeaders.HOST)).thenReturn(host); + when(requestContext.getUriInfo()).thenReturn(uriInfo); + return requestContext; } @Test @@ -101,11 +92,11 @@ public void testVirtualHostStyle() throws Exception { virtualHostStyleFilter.setConfiguration(conf); - ContainerRequest containerRequest = createContainerRequest("mybucket" + - ".localhost:9878", "/myfile", null, true); - virtualHostStyleFilter.filter(containerRequest); + ContainerRequestContext requestContext = createRequestContext( + "mybucket.localhost:9878", "/myfile"); + virtualHostStyleFilter.filter(requestContext); URI expected = new URI("http://" + s3HttpAddr + "/mybucket/myfile"); - assertEquals(expected, containerRequest.getRequestUri()); + verify(requestContext).setRequestUri(new URI("http://" + s3HttpAddr), expected); } @Test @@ -115,12 +106,10 @@ public void testPathStyle() throws Exception { new VirtualHostStyleFilter(); virtualHostStyleFilter.setConfiguration(conf); - ContainerRequest containerRequest = createContainerRequest(s3HttpAddr, - "/mybucket/myfile", null, false); - virtualHostStyleFilter.filter(containerRequest); - URI expected = new URI("http://" + s3HttpAddr + + ContainerRequestContext requestContext = createRequestContext(s3HttpAddr, "/mybucket/myfile"); - assertEquals(expected, containerRequest.getRequestUri()); + virtualHostStyleFilter.filter(requestContext); + verify(requestContext, never()).setRequestUri(any(URI.class), any(URI.class)); } @@ -131,11 +120,11 @@ public void testVirtualHostStyleWithCreateBucketRequest() throws Exception { new VirtualHostStyleFilter(); virtualHostStyleFilter.setConfiguration(conf); - ContainerRequest containerRequest = createContainerRequest("mybucket" + - ".localhost:9878", null, null, true); - virtualHostStyleFilter.filter(containerRequest); + ContainerRequestContext requestContext = createRequestContext( + "mybucket.localhost:9878", null); + virtualHostStyleFilter.filter(requestContext); URI expected = new URI("http://" + s3HttpAddr + "/mybucket"); - assertEquals(expected, containerRequest.getRequestUri()); + verify(requestContext).setRequestUri(new URI("http://" + s3HttpAddr), expected); } @@ -145,11 +134,11 @@ public void testVirtualHostStyleWithCreateKeyRequest() throws Exception { new VirtualHostStyleFilter(); virtualHostStyleFilter.setConfiguration(conf); - ContainerRequest containerRequest = createContainerRequest("mybucket" + - ".localhost:9878", "/key1", null, true); - virtualHostStyleFilter.filter(containerRequest); + ContainerRequestContext requestContext = createRequestContext( + "mybucket.localhost:9878", "/key1"); + virtualHostStyleFilter.filter(requestContext); URI expected = new URI("http://" + s3HttpAddr + "/mybucket/key1"); - assertEquals(expected, containerRequest.getRequestUri()); + verify(requestContext).setRequestUri(new URI("http://" + s3HttpAddr), expected); } @Test @@ -158,19 +147,24 @@ public void testVirtualHostStyleWithQueryParams() throws Exception { VirtualHostStyleFilter virtualHostStyleFilter = new VirtualHostStyleFilter(); virtualHostStyleFilter.setConfiguration(conf); - URI expected = new URI("http://" + s3HttpAddr + "/mybucket?prefix=bh"); - ContainerRequest containerRequest = createContainerRequest("mybucket" + - ".localhost:9878", null, "?prefix=bh", true); - virtualHostStyleFilter.filter(containerRequest); - assertThat(expected.toString()) - .contains(containerRequest.getRequestUri().toString()); + URI baseUri = new URI("http://" + s3HttpAddr); + MultivaluedMap<String, String> queryParams = new MultivaluedHashMap<>(); + queryParams.add("prefix", "bh"); + ContainerRequestContext requestContext = createRequestContext( + "mybucket.localhost:9878", null, queryParams); + virtualHostStyleFilter.filter(requestContext); + verify(requestContext).setRequestUri(baseUri, + new URI("http://" + s3HttpAddr + "/mybucket?prefix=bh")); - containerRequest = createContainerRequest("mybucket" + - ".localhost:9878", null, "?prefix=bh&type=dir", true); - virtualHostStyleFilter.filter(containerRequest); - expected = new URI("http://" + s3HttpAddr + - "/mybucket?prefix=bh&type=dir"); - assertThat(expected.toString()).contains(containerRequest.getRequestUri().toString()); + queryParams.add("type", "dir"); + requestContext = createRequestContext( + "mybucket.localhost:9878", null, queryParams); + virtualHostStyleFilter.filter(requestContext); + ArgumentCaptor<URI> requestUriCaptor = ArgumentCaptor.forClass(URI.class); + verify(requestContext).setRequestUri(eq(baseUri), requestUriCaptor.capture()); + assertThat(requestUriCaptor.getValue().getPath()).isEqualTo("/mybucket"); + assertThat(requestUriCaptor.getValue().getQuery().split("&")) + .containsExactlyInAnyOrder("prefix=bh", "type=dir"); } @@ -180,9 +174,9 @@ public void testVirtualHostStyleWithInvalidInputs(String hostAddress, String expectErrorMessage) throws Exception { VirtualHostStyleFilter virtualHostStyleFilter = new VirtualHostStyleFilter(); virtualHostStyleFilter.setConfiguration(conf); - ContainerRequest containerRequest = createContainerRequest(hostAddress, null, null, true); + ContainerRequestContext requestContext = createRequestContext(hostAddress, null); InvalidRequestException exception = assertThrows(InvalidRequestException.class, - () -> virtualHostStyleFilter.filter(containerRequest)); + () -> virtualHostStyleFilter.filter(requestContext)); assertThat(exception).hasMessageContaining(expectErrorMessage); } }
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketPut.java index 32af79a..e5464e1 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketPut.java
@@ -20,13 +20,19 @@ import static java.net.HttpURLConnection.HTTP_OK; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.BUCKET_ALREADY_EXISTS; +import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.BUCKET_ALREADY_OWNED_BY_YOU; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import javax.ws.rs.core.Response; +import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientStub; +import org.apache.hadoop.ozone.s3.signature.SignatureInfo; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -35,12 +41,16 @@ */ public class TestBucketPut { + private static final String BUCKET_OWNER = "my-s3-owner"; + private static final String OTHER_BUCKET_OWNER = "other-s3-owner"; + private String bucketName = OzoneConsts.BUCKET; private BucketEndpoint bucketEndpoint; + private OzoneClient clientStub; @BeforeEach public void setup() throws Exception { - OzoneClient clientStub = new OzoneClientStub(); + clientStub = new OzoneClientStub(); bucketEndpoint = EndpointBuilder.newBucketEndpointBuilder() .setClient(clientStub) @@ -48,13 +58,53 @@ public void setup() throws Exception { } @Test - public void testCreateBucketAndFailOnDuplicate() throws Exception { + public void testCreateBucketAndFailOnDuplicateWithSameOwner() throws Exception { + BucketEndpoint endpoint = newBucketEndpointWithRequestOwner(BUCKET_OWNER); + clientStub.getObjectStore().createVolume(OzoneConfigKeys.OZONE_S3_VOLUME_NAME_DEFAULT); + clientStub.getObjectStore().getS3Volume().createBucket(bucketName, + BucketArgs.newBuilder().setOwner(BUCKET_OWNER).build()); + + assertErrorResponse(BUCKET_ALREADY_OWNED_BY_YOU, + () -> endpoint.put(bucketName, null)); + } + + @Test + public void testCreateBucketAndFailOnDuplicateWithUnknownRequestOwner() throws Exception { + clientStub.getObjectStore().createVolume(OzoneConfigKeys.OZONE_S3_VOLUME_NAME_DEFAULT); + clientStub.getObjectStore().getS3Volume().createBucket(bucketName); + + assertErrorResponse(BUCKET_ALREADY_EXISTS, + () -> bucketEndpoint.put(bucketName, null)); + } + + @Test + public void testCreateBucketAndFailOnDuplicateWithDifferentOwner() throws Exception { + BucketEndpoint endpoint = newBucketEndpointWithRequestOwner(BUCKET_OWNER); + clientStub.getObjectStore().createVolume(OzoneConfigKeys.OZONE_S3_VOLUME_NAME_DEFAULT); + clientStub.getObjectStore().getS3Volume().createBucket(bucketName, + BucketArgs.newBuilder().setOwner(OTHER_BUCKET_OWNER).build()); + + assertErrorResponse(BUCKET_ALREADY_EXISTS, + () -> endpoint.put(bucketName, null)); + } + + @Test + public void testCreateBucketSuccess() throws Exception { Response response = bucketEndpoint.put(bucketName, null); assertEquals(HTTP_OK, response.getStatus()); assertNotNull(response.getLocation()); + } - // Create-bucket on an existing bucket fails - assertErrorResponse(BUCKET_ALREADY_EXISTS, - () -> bucketEndpoint.put(bucketName, null)); + private BucketEndpoint newBucketEndpointWithRequestOwner(String requestOwner) { + SignatureInfo signatureInfo = mock(SignatureInfo.class); + when(signatureInfo.isSignPayload()).thenReturn(true); + when(signatureInfo.getAwsAccessId()).thenReturn(requestOwner); + when(signatureInfo.getStringToSign()).thenReturn(""); + when(signatureInfo.getSignature()).thenReturn(""); + + return EndpointBuilder.newBucketEndpointBuilder() + .setClient(clientStub) + .setSignatureInfo(signatureInfo) + .build(); } }
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java index e8a8473..741cd76 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
@@ -17,12 +17,15 @@ package org.apache.hadoop.ozone.s3.endpoint; +import static java.nio.charset.StandardCharsets.ISO_8859_1; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertSucceeds; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.get; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.put; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_ARGUMENT; +import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_URI; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_KEY; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.PRECOND_FAILED; import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX; @@ -119,6 +122,24 @@ public void testGetWithNegativePartNumber() throws Exception { } @Test + public void testGetUnreadableKey() { + String unreadableKey = new String(new byte[] {(byte) 0xae, (byte) 0x8a, '-'}, ISO_8859_1); + assertErrorResponse(INVALID_URI, () -> get(rest, BUCKET_NAME, unreadableKey)); + + String malformedUtf8Key = new String(new byte[] {(byte) 0xff}, UTF_8); + assertErrorResponse(INVALID_URI, () -> get(rest, BUCKET_NAME, malformedUtf8Key)); + } + + @Test + public void testGetValidUnicodeKey() throws Exception { + String unicodeKey = "café.txt"; + assertSucceeds(() -> put(rest, BUCKET_NAME, unicodeKey, CONTENT)); + Response response = get(rest, BUCKET_NAME, unicodeKey); + assertEquals(String.valueOf(CONTENT.length()), + response.getHeaderString("Content-Length")); + } + + @Test public void testGet() throws IOException, OS3Exception { //WHEN Response response = get(rest, BUCKET_NAME, KEY_NAME);
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationGet.java index 0558af9..8e4d0eb 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationGet.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationGet.java
@@ -21,6 +21,7 @@ import static java.net.HttpURLConnection.HTTP_OK; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_LIFECYCLE_CONFIGURATION; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.fail; import java.io.ByteArrayInputStream; @@ -124,6 +125,24 @@ public void testGetLifecycleWithBothActions() throws Exception { .getDaysAfterInitiation().intValue()); } + @Test + public void testGetLifecycleWithDateBasedExpiration() throws Exception { + String bucketName = "bucket1"; + bucketEndpoint.put(bucketName, getBodyWithDateExpiration()); + Response r = bucketEndpoint.get(bucketName); + + assertEquals(HTTP_OK, r.getStatus()); + S3LifecycleConfiguration lcc = (S3LifecycleConfiguration) r.getEntity(); + assertEquals(1, lcc.getRules().size()); + S3LifecycleConfiguration.Rule rule = lcc.getRules().get(0); + + assertEquals("expire-on-date", rule.getId()); + assertEquals("prefix/", rule.getPrefix()); + assertEquals("Enabled", rule.getStatus()); + assertEquals("2044-01-19T00:00:00+00:00", rule.getExpiration().getDate()); + assertNull(rule.getExpiration().getDays()); + } + private static InputStream getBody() { String xml = ("<LifecycleConfiguration xmlns=\"http://s3.amazonaws" + ".com/doc/2006-03-01/\">" + @@ -153,6 +172,19 @@ private static InputStream getBodyWithAbortAction() { return new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)); } + private static InputStream getBodyWithDateExpiration() { + String xml = "<LifecycleConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">" + + "<Rule>" + + "<ID>expire-on-date</ID>" + + "<Prefix>prefix/</Prefix>" + + "<Status>Enabled</Status>" + + "<Expiration><Date>2044-01-19T00:00:00+00:00</Date></Expiration>" + + "</Rule>" + + "</LifecycleConfiguration>"; + + return new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)); + } + private static InputStream getBodyWithBothActions() { String xml = "<LifecycleConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">" + "<Rule>" +
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationPut.java index d6274b6..818df7f 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationPut.java
@@ -19,15 +19,22 @@ import static java.net.HttpURLConnection.HTTP_BAD_REQUEST; import static java.net.HttpURLConnection.HTTP_FORBIDDEN; +import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.net.HttpURLConnection.HTTP_OK; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.ACCESS_DENIED; +import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INTERNAL_ERROR; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_REQUEST; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.MALFORMED_XML; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_BUCKET; +import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.QUOTA_EXCEEDED; import static org.apache.hadoop.ozone.s3.util.S3Consts.EXPECTED_BUCKET_OWNER_HEADER; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -38,8 +45,13 @@ import javax.ws.rs.core.HttpHeaders; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.ObjectStore; +import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientStub; +import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.client.protocol.ClientProtocol; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.util.S3Consts; import org.junit.jupiter.api.BeforeEach; @@ -182,6 +194,54 @@ public void testPutLifecycleConfigurationFailsWithNonBucketOwner() } @Test + public void testPutLifecycleConfigurationPropagatesQuotaExceeded() + throws Exception { + // QUOTA_EXCEEDED is not explicitly handled by putBucketLifecycleConfiguration. + // It must still surface as an OS3Exception instead of a false HTTP 200. + assertUnhandledOMExceptionPropagated( + new OMException("Quota exceeded", OMException.ResultCodes.QUOTA_EXCEEDED), + HTTP_FORBIDDEN, QUOTA_EXCEEDED.getCode()); + } + + @Test + public void testPutLifecycleConfigurationPropagatesInternalError() + throws Exception { + // An unexpected internal OM failure must be reported as InternalError (HTTP 500), + // not swallowed into a false HTTP 200 success. + assertUnhandledOMExceptionPropagated( + new OMException("boom", OMException.ResultCodes.INTERNAL_ERROR), + HTTP_INTERNAL_ERROR, INTERNAL_ERROR.getCode()); + } + + private void assertUnhandledOMExceptionPropagated(OMException omException, + int expectedHttpCode, String expectedErrorCode) throws Exception { + OzoneClient mockClient = mock(OzoneClient.class); + ObjectStore mockObjectStore = mock(ObjectStore.class); + OzoneVolume mockVolume = mock(OzoneVolume.class); + OzoneBucket mockBucket = mock(OzoneBucket.class); + when(mockBucket.getVolumeName()).thenReturn("s3v"); + when(mockBucket.getName()).thenReturn("bucket1"); + when(mockBucket.getBucketLayout()).thenReturn(BucketLayout.OBJECT_STORE); + doThrow(omException).when(mockBucket).setLifecycleConfiguration(any()); + ClientProtocol clientProtocol = mock(ClientProtocol.class); + when(mockClient.getObjectStore()).thenReturn(mockObjectStore); + when(mockClient.getProxy()).thenReturn(clientProtocol); + when(mockObjectStore.getClientProxy()).thenReturn(clientProtocol); + when(mockObjectStore.getS3Volume()).thenReturn(mockVolume); + when(mockVolume.getBucket("bucket1")).thenReturn(mockBucket); + + BucketEndpoint endpoint = EndpointBuilder.newBucketEndpointBuilder() + .setClient(mockClient) + .build(); + endpoint.queryParamsForTest().set(S3Consts.QueryParams.LIFECYCLE, ""); + + OS3Exception ex = assertThrows(OS3Exception.class, + () -> endpoint.put("bucket1", onePrefix())); + assertEquals(expectedHttpCode, ex.getHttpCode()); + assertEquals(expectedErrorCode, ex.getCode()); + } + + @Test public void testPutLifecycleWithAbortIncompleteMultipartUpload() throws Exception { assertEquals(HTTP_OK, bucketEndpoint.put("bucket1", withAbortIncompleteMultipartUpload()).getStatus()); }
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java index 6d1d4d1..0d5e6ec 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java
@@ -17,26 +17,30 @@ package org.apache.hadoop.ozone.local; +import java.io.PrintWriter; import java.nio.file.Path; import java.time.Duration; import java.time.format.DateTimeParseException; import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import org.apache.hadoop.hdds.cli.AbstractSubcommand; import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.TimeDurationUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import picocli.CommandLine; import picocli.CommandLine.Command; import picocli.CommandLine.ITypeConverter; import picocli.CommandLine.Option; /** - * Internal CLI entry point for local Ozone cluster commands. + * CLI entry point for local single-node Ozone. */ @Command(name = "ozone local", - hidden = true, - description = "Internal commands for local Ozone cluster runtime", + description = "Run a single-node local Ozone cluster", versionProvider = HddsVersionProvider.class, mixinStandardHelpOptions = true, subcommands = { @@ -44,6 +48,8 @@ }) public class OzoneLocal extends GenericCli { + private static final Logger LOG = LoggerFactory.getLogger(OzoneLocal.class); + static final String ENV_DATA_DIR = "OZONE_LOCAL_DATA_DIR"; static final String ENV_FORMAT = "OZONE_LOCAL_FORMAT"; static final String ENV_DATANODES = "OZONE_LOCAL_DATANODES"; @@ -108,8 +114,7 @@ public static void main(String[] args) { } @Command(name = "run", - hidden = true, - description = "Resolve configuration for local Ozone runtime startup") + description = "Start SCM, OM, and datanodes in one local process") static class RunCommand extends AbstractSubcommand implements Callable<Void> { @Option(names = "--data-dir", @@ -189,11 +194,56 @@ static class RunCommand extends AbstractSubcommand implements Callable<Void> { private String s3Region; @Override - public Void call() { - resolveConfig(); + public Void call() throws Exception { + LocalOzoneClusterConfig config = resolveConfig(); + try (LocalOzoneRuntime runtime = createRuntime(config, getOzoneConf())) { + runtime.start(); + printSummary(runtime, config); + awaitShutdown(runtime); + } return null; } + LocalOzoneRuntime createRuntime(LocalOzoneClusterConfig config, OzoneConfiguration seedConfiguration) { + return new LocalOzoneCluster(config, seedConfiguration); + } + + private void printSummary(LocalOzoneRuntime runtime, LocalOzoneClusterConfig config) { + PrintWriter writer = out(); + writer.println("Local Ozone is running from " + config.getDataDir()); + writer.println("SCM RPC: " + runtime.getDisplayHost() + ":" + runtime.getScmPort()); + writer.println("OM RPC: " + runtime.getDisplayHost() + ":" + runtime.getOmPort()); + writer.println("Press Ctrl+C to stop."); + writer.flush(); + } + + /** + * Blocks until the JVM is asked to shut down (for example by Ctrl+C), closing the runtime from + * the shutdown hook before the JVM exits. + */ + void awaitShutdown(LocalOzoneRuntime runtime) throws InterruptedException { + CountDownLatch stopped = new CountDownLatch(1); + Thread shutdownHook = new Thread(() -> { + try { + runtime.close(); + } catch (Exception ex) { + LOG.warn("Failed to close ozone local runtime", ex); + } finally { + stopped.countDown(); + } + }, "ozone-local-shutdown"); + Runtime.getRuntime().addShutdownHook(shutdownHook); + try { + stopped.await(); + } finally { + try { + Runtime.getRuntime().removeShutdownHook(shutdownHook); + } catch (IllegalStateException ignored) { + // JVM shutdown is already in progress. + } + } + } + LocalOzoneClusterConfig resolveConfig() { if (datanodes < 1) { throw new IllegalArgumentException(
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java index 219b3ae..0f8686e 100644 --- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java
@@ -30,6 +30,7 @@ import java.lang.reflect.Field; import java.nio.file.Paths; import java.time.Duration; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.junit.jupiter.api.Test; import picocli.CommandLine; import picocli.CommandLine.Command; @@ -45,21 +46,21 @@ class TestOzoneLocal { @Test - void localCommandMetadataIsPresentAndHidden() { + void localCommandMetadataIsPresentAndPublic() { Command command = OzoneLocal.class.getAnnotation(Command.class); assertNotNull(command); assertEquals("ozone local", command.name()); - assertTrue(command.hidden()); + assertFalse(command.hidden()); } @Test - void runCommandMetadataIsPresentAndHidden() { + void runCommandMetadataIsPresentAndPublic() { Command command = OzoneLocal.RunCommand.class.getAnnotation(Command.class); assertNotNull(command); assertEquals("run", command.name()); - assertTrue(command.hidden()); + assertFalse(command.hidden()); } @Test @@ -70,7 +71,7 @@ void genericCliRegistersRunCommand() { } @Test - void rootHelpHidesRunCommand() throws Exception { + void rootHelpListsRunCommand() throws Exception { OzoneLocal local = new OzoneLocal(); CommandLine commandLine = local.getCmd(); ByteArrayOutputStream out = new ByteArrayOutputStream(); @@ -85,27 +86,41 @@ void rootHelpHidesRunCommand() throws Exception { String help = out.toString(UTF_8.name()); assertEquals(0, exitCode); assertTrue(help.contains("Usage: ozone local")); - assertFalse(help.matches("(?s).*\\R\\s+run\\b.*"), help); + assertTrue(help.matches("(?s).*\\R\\s+run\\b.*"), help); assertEquals("", err.toString(UTF_8.name())); } @Test - void runCommandResolvesConfigurationQuietlyUntilRuntimeStartup() - throws Exception { - OzoneLocal local = new OzoneLocal(); - CommandLine commandLine = local.getCmd(); + void runCommandStartsRuntimeAndPrintsStartupSummary() throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); - ByteArrayOutputStream err = new ByteArrayOutputStream(); + StubRuntime runtime = new StubRuntime("localhost", 9860, 9862); + TestableRunCommand command = new TestableRunCommand(runtime); + CommandLine commandLine = new CommandLine(command); commandLine.setOut(new PrintWriter(new OutputStreamWriter(out, UTF_8), true)); - commandLine.setErr(new PrintWriter(new OutputStreamWriter(err, UTF_8), - true)); - int exitCode = local.execute(new String[] {"run"}); + int exitCode = commandLine.execute(); assertEquals(0, exitCode); - assertEquals("", out.toString(UTF_8.name())); - assertEquals("", err.toString(UTF_8.name())); + assertTrue(runtime.started); + assertTrue(runtime.closed); + String text = out.toString(UTF_8.name()); + assertTrue(text.contains("Local Ozone is running from"), text); + assertTrue(text.contains("SCM RPC: localhost:9860"), text); + assertTrue(text.contains("OM RPC: localhost:9862"), text); + assertTrue(text.contains("Press Ctrl+C to stop."), text); + } + + @Test + void runCommandClosesRuntimeWhenStartupFails() { + StubRuntime runtime = new StubRuntime("localhost", 9860, 9862); + runtime.failStart = true; + TestableRunCommand command = new TestableRunCommand(runtime); + + int exitCode = new CommandLine(command).execute(); + + assertEquals(1, exitCode); + assertTrue(runtime.closed); } @Test @@ -323,6 +338,79 @@ private static void assertEnvDefault(String fieldName, defaultValue); } + private static final class TestableRunCommand extends OzoneLocal.RunCommand { + + private final LocalOzoneRuntime runtime; + + private TestableRunCommand(LocalOzoneRuntime runtime) { + this.runtime = runtime; + } + + @Override + LocalOzoneRuntime createRuntime(LocalOzoneClusterConfig config, OzoneConfiguration seedConfiguration) { + return runtime; + } + + @Override + void awaitShutdown(LocalOzoneRuntime localRuntime) { + // Return immediately instead of blocking until JVM shutdown. + } + } + + private static final class StubRuntime implements LocalOzoneRuntime { + + private final String displayHost; + private final int scmPort; + private final int omPort; + private boolean failStart; + private boolean started; + private boolean closed; + + private StubRuntime(String displayHost, int scmPort, int omPort) { + this.displayHost = displayHost; + this.scmPort = scmPort; + this.omPort = omPort; + } + + @Override + public void start() { + if (failStart) { + throw new IllegalStateException("startup failed"); + } + started = true; + } + + @Override + public String getDisplayHost() { + return displayHost; + } + + @Override + public int getScmPort() { + return scmPort; + } + + @Override + public int getOmPort() { + return omPort; + } + + @Override + public int getS3gPort() { + return 0; + } + + @Override + public String getS3Endpoint() { + return ""; + } + + @Override + public void close() { + closed = true; + } + } + private static final class RunCommandFallbackDefaults implements IDefaultValueProvider {
diff --git a/pom.xml b/pom.xml index 65c8f23..ef4dd07 100644 --- a/pom.xml +++ b/pom.xml
@@ -41,7 +41,7 @@ <assertj.version>3.27.7</assertj.version> <aws-java-sdk.version>1.12.788</aws-java-sdk.version> <bonecp.version>0.8.0.RELEASE</bonecp.version> - <bouncycastle.version>1.84</bouncycastle.version> + <bouncycastle.version>1.85</bouncycastle.version> <build-helper-maven-plugin.version>3.6.1</build-helper-maven-plugin.version> <byte-buddy.version>1.18.11</byte-buddy.version> <cdi-api.version>2.0.SP1</cdi-api.version> @@ -147,7 +147,7 @@ <maven-shade-plugin.version>3.6.2</maven-shade-plugin.version> <maven-site-plugin.version>3.22.0</maven-site-plugin.version> <maven-source-plugin.version>3.4.0</maven-source-plugin.version> - <maven-surefire-plugin.argLine>-Xmx8192m -XX:+HeapDumpOnOutOfMemoryError</maven-surefire-plugin.argLine> + <maven-surefire-plugin.argLine>-Xmx8192m -XX:+HeapDumpOnOutOfMemoryError -XX:+IgnoreUnrecognizedVMOptions -XX:+EnableDynamicAgentLoading</maven-surefire-plugin.argLine> <maven-surefire-plugin.argLineAccessArgs /> <maven-surefire-plugin.version>3.5.2</maven-surefire-plugin.version> <maven-surefire-report-plugin.version>${maven-surefire-plugin.version}</maven-surefire-report-plugin.version> @@ -161,7 +161,7 @@ <!-- Node.js version number (without the v prefix required by frontend-maven-plugin) --> <nodejs.version>24.17.0</nodejs.version> <okhttp3.version>4.12.0</okhttp3.version> - <opentelemetry.version>1.63.0</opentelemetry.version> + <opentelemetry.version>1.64.0</opentelemetry.version> <os-maven-plugin.version>1.7.1</os-maven-plugin.version> <ozone.release>Kenai Fjords</ozone.release> <ozone.shaded.native.prefix>org_apache_ozone_shaded</ozone.shaded.native.prefix>