Syncing HDDS-3001 branch with master (#950)
diff --git a/.github/comment-commands/close.sh b/.github/comment-commands/close.sh
new file mode 100755
index 0000000..4624bd8
--- /dev/null
+++ b/.github/comment-commands/close.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#doc: Close pending pull request temporary
+# shellcheck disable=SC2124
+MESSAGE="Thank you very much for the patch. I am closing this PR __temporarily__ as there was no
+activity recently and it is waiting for response from its author.
+
+It doesn't mean that this PR is not important or ignored: feel free to reopen the PR at any time.
+
+It only means that attention of committers is not required. We prefer to keep the review queue clean. This ensures PRs in need of review are more visible, which results in faster feedback for all PRs.
+
+If you need ANY help to finish this PR, please [contact the community](https://github.com/apache/hadoop-ozone#contact) on the mailing list or the slack channel."
+
+set +x #GITHUB_TOKEN
+curl -s -o /dev/null \
+ -X POST \
+ --data "$(jq --arg body "$MESSAGE" -n '{body: $body}')" \
+ --header "authorization: Bearer $GITHUB_TOKEN" \
+ --header 'content-type: application/json' \
+ "$(jq -r '.issue.comments_url' "$GITHUB_EVENT_PATH")"
+
+curl -s -o /dev/null \
+ -X PATCH \
+ --data '{"state": "close"}' \
+ --header "authorization: Bearer $GITHUB_TOKEN" \
+ --header 'content-type: application/json' \
+ "$(jq -r '.issue.pull_request.url' "$GITHUB_EVENT_PATH")"
diff --git a/.github/comment-commands/pending.sh b/.github/comment-commands/pending.sh
index 995e37f..08947f6 100755
--- a/.github/comment-commands/pending.sh
+++ b/.github/comment-commands/pending.sh
@@ -29,3 +29,10 @@
--header "authorization: Bearer $GITHUB_TOKEN" \
--header 'content-type: application/json' \
"$URL"
+
+curl -s -o /dev/null \
+ -X POST \
+ --data '{"labels": [ "pending" ]}' \
+ --header "authorization: Bearer $GITHUB_TOKEN" \
+ "$(jq -r '.issue.url' "$GITHUB_EVENT_PATH")/labels"
+
diff --git a/.github/comment-commands/ready.sh b/.github/comment-commands/ready.sh
index 55087a8..0abbc3e 100755
--- a/.github/comment-commands/ready.sh
+++ b/.github/comment-commands/ready.sh
@@ -27,3 +27,9 @@
--data "$(jq --arg message "$MESSAGE" -n '{message: $message}')" \
--header "authorization: Bearer $GITHUB_TOKEN" \
"$URL"/ISSUE_ID/dismissals
+
+curl -s -o /dev/null \
+ -X DELETE \
+ --header "authorization: Bearer $GITHUB_TOKEN" \
+ "$(jq -r '.issue.url' "$GITHUB_EVENT_PATH")/labels/pending"
+
diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml
index 1b9a446..5a920e5 100644
--- a/.github/workflows/post-commit.yml
+++ b/.github/workflows/post-commit.yml
@@ -104,102 +104,126 @@
- uses: ./.github/buildenv
with:
args: ./hadoop-ozone/dev-support/checks/build.sh
+ # remove image created for 'buildenv'
+ - run: docker image rm $(docker images -a -q | head -1) || true
+ # remove its big parent build image
+ - run: docker image rm apache/ozone-build || true
- run: sudo pip install robotframework
- - run: sudo chown runner -R .
- - run: cd ./hadoop-ozone/dist/target/ozone-*/ && mkdir .aws && sudo chown 1000 .aws
- - run: ./hadoop-ozone/dev-support/checks/acceptance.sh
+ - run: sudo mv ./hadoop-ozone/dist/target/ozone-* /mnt/ozone && sudo chown runner -R /mnt/ozone
+ - run: cd /mnt/ozone && mkdir .aws && sudo chown 1000 .aws
+ - run: cd /mnt/ozone/compose && ./test-all.sh
+ env:
+ KEEP_IMAGE: false
- uses: actions/upload-artifact@master
if: always()
with:
name: acceptance
- path: target/acceptance
+ path: /mnt/ozone/compose/result
it-freon:
name: it-freon
runs-on: ubuntu-18.04
needs:
- build
steps:
+ - run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt
- uses: actions/checkout@master
- - uses: ./.github/buildenv
with:
- args: ./hadoop-ozone/dev-support/checks/integration.sh -Pfreon
+ path: mnt/ozone
+ - uses: ./mnt/ozone/.github/buildenv
+ with:
+ args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -Pfreon
- uses: actions/upload-artifact@master
if: always()
with:
name: it-freon
- path: target/integration
+ path: mnt/ozone/target/integration
it-filesystem:
name: it-filesystem
runs-on: ubuntu-18.04
needs:
- build
steps:
+ - run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt
- uses: actions/checkout@master
- - uses: ./.github/buildenv
with:
- args: ./hadoop-ozone/dev-support/checks/integration.sh -Pfilesystem
+ path: mnt/ozone
+ - uses: ./mnt/ozone/.github/buildenv
+ with:
+ args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -Pfilesystem
- uses: actions/upload-artifact@master
if: always()
with:
name: it-filesystem
- path: target/integration
+ path: mnt/ozone/target/integration
it-filesystem-contract:
name: it-filesystem-contract
runs-on: ubuntu-18.04
needs:
- build
steps:
+ - run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt
- uses: actions/checkout@master
- - uses: ./.github/buildenv
with:
- args: ./hadoop-ozone/dev-support/checks/integration.sh -Pfilesystem-contract
+ path: mnt/ozone
+ - uses: ./mnt/ozone/.github/buildenv
+ with:
+ args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -Pfilesystem-contract
- uses: actions/upload-artifact@master
if: always()
with:
name: it-filesystem-contract
- path: target/integration
+ path: mnt/ozone/target/integration
it-client:
name: it-client
runs-on: ubuntu-18.04
needs:
- build
steps:
+ - run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt
- uses: actions/checkout@master
- - uses: ./.github/buildenv
with:
- args: ./hadoop-ozone/dev-support/checks/integration.sh -Pclient
+ path: mnt/ozone
+ - uses: ./mnt/ozone/.github/buildenv
+ with:
+ args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -Pclient
- uses: actions/upload-artifact@master
if: always()
with:
name: it-client
- path: target/integration
+ path: mnt/ozone/target/integration
it-hdds-om:
name: it-hdds-om
runs-on: ubuntu-18.04
needs:
- build
steps:
+ - run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt
- uses: actions/checkout@master
- - uses: ./.github/buildenv
with:
- args: ./hadoop-ozone/dev-support/checks/integration.sh -Phdds-om
+ path: mnt/ozone
+ - uses: ./mnt/ozone/.github/buildenv
+ with:
+ args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -Phdds-om
- uses: actions/upload-artifact@master
if: always()
with:
name: it-hdds-om
- path: target/integration
+ path: mnt/ozone/target/integration
it-ozone:
name: it-ozone
runs-on: ubuntu-18.04
needs:
- build
steps:
+ - run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt
- uses: actions/checkout@master
- - uses: ./.github/buildenv
with:
- args: ./hadoop-ozone/dev-support/checks/integration.sh -Pozone
+ path: mnt/ozone
+ - uses: ./mnt/ozone/.github/buildenv
+ with:
+ args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -Pozone
- uses: actions/upload-artifact@master
if: always()
with:
name: it-ozone
- path: target/integration
+ path: mnt/ozone/target/integration
\ No newline at end of file
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 119dd0c..f9a4230 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -97,87 +97,126 @@
- uses: ./.github/buildenv
with:
args: ./hadoop-ozone/dev-support/checks/build.sh
+ # remove image created for 'buildenv'
+ - run: docker image rm $(docker images -a -q | head -1) || true
+ # remove its big parent build image
+ - run: docker image rm apache/ozone-build || true
- run: sudo pip install robotframework
- - run: sudo chown runner -R .
- - run: cd ./hadoop-ozone/dist/target/ozone-*/ && mkdir .aws && sudo chown 1000 .aws
- - run: ./hadoop-ozone/dev-support/checks/acceptance.sh
+ - run: sudo mv ./hadoop-ozone/dist/target/ozone-* /mnt/ozone && sudo chown runner -R /mnt/ozone
+ - run: cd /mnt/ozone && mkdir .aws && sudo chown 1000 .aws
+ - run: cd /mnt/ozone/compose && ./test-all.sh
+ env:
+ KEEP_IMAGE: false
- uses: actions/upload-artifact@master
if: always()
with:
name: acceptance
- path: target/acceptance
+ path: /mnt/ozone/compose/result
+ it-freon:
+ name: it-freon
+ runs-on: ubuntu-18.04
+ needs:
+ - build
+ steps:
+ - run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt
+ - uses: actions/checkout@master
+ with:
+ path: mnt/ozone
+ - uses: ./mnt/ozone/.github/buildenv
+ with:
+ args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -Pfreon
+ - uses: actions/upload-artifact@master
+ if: always()
+ with:
+ name: it-freon
+ path: mnt/ozone/target/integration
it-filesystem:
name: it-filesystem
runs-on: ubuntu-18.04
needs:
- build
steps:
+ - run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt
- uses: actions/checkout@master
- - uses: ./.github/buildenv
with:
- args: ./hadoop-ozone/dev-support/checks/integration.sh -Pfilesystem
+ path: mnt/ozone
+ - uses: ./mnt/ozone/.github/buildenv
+ with:
+ args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -Pfilesystem
- uses: actions/upload-artifact@master
if: always()
with:
name: it-filesystem
- path: target/integration
+ path: mnt/ozone/target/integration
it-filesystem-contract:
name: it-filesystem-contract
runs-on: ubuntu-18.04
needs:
- build
steps:
+ - run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt
- uses: actions/checkout@master
- - uses: ./.github/buildenv
with:
- args: ./hadoop-ozone/dev-support/checks/integration.sh -Pfilesystem-contract
+ path: mnt/ozone
+ - uses: ./mnt/ozone/.github/buildenv
+ with:
+ args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -Pfilesystem-contract
- uses: actions/upload-artifact@master
if: always()
with:
name: it-filesystem-contract
- path: target/integration
+ path: mnt/ozone/target/integration
it-client:
name: it-client
runs-on: ubuntu-18.04
needs:
- build
steps:
+ - run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt
- uses: actions/checkout@master
- - uses: ./.github/buildenv
with:
- args: ./hadoop-ozone/dev-support/checks/integration.sh -Pclient
+ path: mnt/ozone
+ - uses: ./mnt/ozone/.github/buildenv
+ with:
+ args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -Pclient
- uses: actions/upload-artifact@master
if: always()
with:
name: it-client
- path: target/integration
+ path: mnt/ozone/target/integration
it-hdds-om:
name: it-hdds-om
runs-on: ubuntu-18.04
needs:
- build
steps:
+ - run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt
- uses: actions/checkout@master
- - uses: ./.github/buildenv
with:
- args: ./hadoop-ozone/dev-support/checks/integration.sh -Phdds-om
+ path: mnt/ozone
+ - uses: ./mnt/ozone/.github/buildenv
+ with:
+ args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -Phdds-om
- uses: actions/upload-artifact@master
if: always()
with:
name: it-hdds-om
- path: target/integration
+ path: mnt/ozone/target/integration
it-ozone:
name: it-ozone
runs-on: ubuntu-18.04
needs:
- build
steps:
+ - run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt
- uses: actions/checkout@master
- - uses: ./.github/buildenv
with:
- args: ./hadoop-ozone/dev-support/checks/integration.sh -Pozone
+ path: mnt/ozone
+ - uses: ./mnt/ozone/.github/buildenv
+ with:
+ args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -Pozone
- uses: actions/upload-artifact@master
if: always()
with:
name: it-ozone
- path: target/integration
+ path: mnt/ozone/target/integration
\ No newline at end of file
diff --git a/CONTRIBUTION.md b/CONTRIBUTING.md
similarity index 94%
rename from CONTRIBUTION.md
rename to CONTRIBUTING.md
index 622acd2..5dd59d3 100644
--- a/CONTRIBUTION.md
+++ b/CONTRIBUTING.md
@@ -44,7 +44,6 @@
* Unix System
* JDK 1.8 or higher
* Maven 3.5 or later
-* Protocol Buffers 2.5 (see the next session for installation)
* Internet connection for first build (to fetch all Maven and Hadoop dependencies)
Additional requirements to run your first pseudo cluster:
@@ -63,28 +62,11 @@
* [hugo](https://gohugo.io/) to include the documentation in the web ui.
-#### Installing protobuf 2.5
-
-Protobuf 2.5 can be installed from the source:
-
-```
-mkdir -p /usr/local/src/
-cd /usr/local/src/
-wget https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz
-tar xvf protobuf-2.5.0.tar.gz
-cd protobuf-2.5.0
-./autogen.sh
-./configure --prefix=/usr
-make
-make install
-protoc --version
-```
-
(Standard development tools such as make, gcc, etc. are required.)
### Build the project
-After installing the requirements (especially maven and protobuf) the build is as simple as:
+After installing the requirements (especially maven) the build is as simple as:
```
mvn clean install -DskipTests
diff --git a/HISTORY.md b/HISTORY.md
new file mode 100644
index 0000000..233471c
--- /dev/null
+++ b/HISTORY.md
@@ -0,0 +1,61 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# History of Apache Hadoop Ozone project
+
+Ozone development was started on a feature branch HDFS-7240 as part of the Apache Hadoop HDFS project. Based on the Jira information the first Ozone commit was the commit of [HDFS-8456 Ozone: Introduce STORAGE_CONTAINER_SERVICE as a new NodeType.](https://issues.apache.org/jira/browse/HDFS-8456) in May 2015.
+
+
+Ozone is an Object Store for Hadoop which is based on a lower level storage replication layer. This layer was originally called HDSL (Hadoop Distributed Storage Layer) and later renamed to HDDS (Hadoop Distributed Data Storage).
+
+Implementation of the generic storage layer began under [HDFS-11118](https://issues.apache.org/jira/browse/HDFS-11118) together with a iScsi/[jScsi](https://github.com/sebastiangraf/jSCSI) based block storage layer ("CBlock") introduced by [HDFS-11361](https://issues.apache.org/jira/browse/HDFS-11361).
+
+As a summary:
+
+ * HDDS (earlier HDSL): replicates huge binary _containers_ between datanodes
+ * Ozone: provides Object Store semantics with the help of HDDS
+ * CBlock: provides mountable volumes with the help of the HDDS layer (based on iScsi protocol)
+
+In the beginning of the year 2017 a new podling project was started inside [Apache Incubator](http://incubator.apache.org/): [Apache Ratis](https://ratis.apache.org/). Ratis is an embeddable RAFT protcol implementation it is which became the corner stone of consensus inside both Ozone and HDDS projects. (Started to [be used](https://issues.apache.org/jira/browse/HDFS-11519) by Ozone in March of 2017)
+
+In the October of 2017 a [discussion](https://lists.apache.org/thread.html/3b5b65ce428f88299e6cb4c5d745ec65917490be9e417d361cc08d7e@%3Chdfs-dev.hadoop.apache.org%3E) has been started on hdfs-dev mailing list to merge the existing functionality to the Apache Hadoop trunk. After a long debate Owen O'Malley [suggested a consensus](https://lists.apache.org/thread.html/c85e5263dcc0ca1d13cbbe3bcfb53236784a39111b8c353f60582eb4@%3Chdfs-dev.hadoop.apache.org%3E) to merge it to the trunk but use separated release cycle:
+
+ > * HDSL become a subproject of Hadoop.
+ > * HDSL will release separately from Hadoop. Hadoop releases will notcontain HDSL and vice versa.
+ > * HDSL will get its own jira instance so that the release tags stay separate.
+ > * On trunk (as opposed to release branches) HDSL will be a separate module in Hadoop's source tree. This will enable the HDSL to work on their trunk and the Hadoop trunk without making releases for every change.
+ > * Hadoop's trunk will only build HDSL if a non-default profile is enabled. When Hadoop creates a release branch, the RM will delete the HDSL module from the branch.
+ > * HDSL will have their own Yetus checks and won't cause failures in the Hadoop patch check.
+
+This proposal was passed and after reorganizing the code (see HDFS-13258) and Ozone [has been voted](https://lists.apache.org/thread.html/ad0fe160ae84be97a0a87865059761ad7cd747be7b2fe060707d4f28@%3Chdfs-dev.hadoop.apache.org%3E) to be merged to the Hadoop trunk at the March of 2018.
+
+As the CBlock feature was not stable enough it was not merged and archived on a separated feature branch which was not synced with the newer Ozone/HDDS features. (Somewhat similar functionality is provided later with S3 Fuse file system and an S3 compatible REST gateway.)
+
+After the merge a new Jira project was created (HDDS) and the work was tracked under that project instead of child issues under HDFS-7240.
+
+In the next year multiple Ozone releases has been published in separated release package. The Ozone source release was developed on the Hadoop trunk, but the Ozone sources are removed from the main Hadoop releases.
+
+Originally, Ozone depended on the in-tree (SNAPSHOT) Hadoop artifacts. It was required to compile the core hadoop-hdfs/hadoop-common artifacts before compiling the Ozone subprojects. During the development this dependency was reduced more and more. With the 0.4.1 release this dependency has been totally removed and it became possible to compile Ozone with the help of the released Hadoop artifacts which made it possible to separate the development of Ozone from the main Hadoop trunk branch.
+
+In October 2019, the Ozone sources were moved out to the [apache/hadoop-ozone](https://github.com/apache/hadoop-ozone) git repository. During this move the git history was transformed to remove old YARN/HDFS/MAPREDUCE tasks.
+
+ * The first git commit of the new repository is the commit which created the new maven subprojects for Ozone (before the trunk merge)
+ * Some of the oldest Ozone commits are available only from the Hadoop repository.
+ * Some newer HDDS commits have different commit hash in `hadoop` and `hadoop-ozone` repository.
+
+
+In March 2020, [Ozone 0.5.0 was released](https://hadoop.apache.org/ozone/release/0.5.0-beta/), the first release marked as _beta_tag (earlier releases were alpha).
diff --git a/README.md b/README.md
index 391cfa2..ddf4c6e 100644
--- a/README.md
+++ b/README.md
@@ -82,16 +82,16 @@
cd hadoop-ozone/dist/target/ozone-*/compose/ozone
docker-compose up -d --scale datanode=3
```
-For more information, you can check the [Contribution guideline](./CONTRIBUTION.md)
+For more information, you can check the [Contribution guideline](./CONTRIBUTING.md)
## Contribute
All contributions are welcome.
- 1. Please open a [Jira](https://issues.apache.org/jira) issue
+ 1. Please open a [Jira](https://issues.apache.org/jira/projects/HDDS/issues) issue
2. And create a pull request
-For more information, you can check the [Contribution guideline](./CONTRIBUTION.md)
+For more information, you can check the [Contribution guideline](./CONTRIBUTING.md)
## License
diff --git a/dev-support/bin/qbt b/dev-support/bin/qbt
deleted file mode 100755
index fe5e6f6..0000000
--- a/dev-support/bin/qbt
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-BINDIR=$(cd -P -- "$(dirname -- "${BASH_SOURCE-0}")" >/dev/null && pwd -P)
-exec "${BINDIR}/yetus-wrapper" qbt --project=hadoop --skip-dir=dev-support "$@"
diff --git a/dev-support/bin/smart-apply-patch b/dev-support/bin/smart-apply-patch
deleted file mode 100755
index 3fd469f..0000000
--- a/dev-support/bin/smart-apply-patch
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-BINDIR=$(cd -P -- "$(dirname -- "${BASH_SOURCE-0}")" >/dev/null && pwd -P)
-exec "${BINDIR}/yetus-wrapper" smart-apply-patch --project=hadoop "$@"
diff --git a/dev-support/bin/test-patch b/dev-support/bin/test-patch
deleted file mode 100755
index 8ff8119..0000000
--- a/dev-support/bin/test-patch
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-BINDIR=$(cd -P -- "$(dirname -- "${BASH_SOURCE-0}")" >/dev/null && pwd -P)
-exec "${BINDIR}/yetus-wrapper" test-patch --project=hadoop --skip-dir=dev-support "$@"
diff --git a/dev-support/bin/yetus-wrapper b/dev-support/bin/yetus-wrapper
deleted file mode 100755
index b0f71f1..0000000
--- a/dev-support/bin/yetus-wrapper
+++ /dev/null
@@ -1,188 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# you must be this high to ride the ride
-if [[ -z "${BASH_VERSINFO[0]}" ]] \
- || [[ "${BASH_VERSINFO[0]}" -lt 3 ]] \
- || [[ "${BASH_VERSINFO[0]}" -eq 3 && "${BASH_VERSINFO[1]}" -lt 2 ]]; then
- echo "bash v3.2+ is required. Sorry."
- exit 1
-fi
-
-set -o pipefail
-
-## @description Print a message to stderr
-## @audience public
-## @stability stable
-## @replaceable no
-## @param string
-function yetus_error
-{
- echo "$*" 1>&2
-}
-
-## @description Given a filename or dir, return the absolute version of it
-## @audience public
-## @stability stable
-## @param directory
-## @replaceable no
-## @return 0 success
-## @return 1 failure
-## @return stdout abspath
-function yetus_abs
-{
- declare obj=$1
- declare dir
- declare fn
- declare dirret
-
- if [[ ! -e ${obj} ]]; then
- return 1
- elif [[ -d ${obj} ]]; then
- dir=${obj}
- else
- dir=$(dirname -- "${obj}")
- fn=$(basename -- "${obj}")
- fn="/${fn}"
- fi
-
- dir=$(cd -P -- "${dir}" >/dev/null 2>/dev/null && pwd -P)
- dirret=$?
- if [[ ${dirret} = 0 ]]; then
- echo "${dir}${fn}"
- return 0
- fi
- return 1
-}
-
-function version_ge()
-{
- test "$(echo "$@" | tr " " "\n" | sort -rV | head -n 1)" == "$1";
-}
-
-WANTED="$1"
-shift
-ARGV=("$@")
-
-HADOOP_YETUS_VERSION=${HADOOP_YETUS_VERSION:-0.10.0}
-BIN=$(yetus_abs "${BASH_SOURCE-$0}")
-BINDIR=$(dirname "${BIN}")
-
-## HADOOP_YETUS_VERSION >= 0.9.0 the tarball named with apache-yetus prefix
-if version_ge "${HADOOP_YETUS_VERSION}" "0.9.0"; then
- YETUS_PREFIX=apache-yetus
-else
- YETUS_PREFIX=yetus
-fi
-
-###
-### if YETUS_HOME is set, then try to use it
-###
-if [[ -n "${YETUS_HOME}" && -x "${YETUS_HOME}/bin/${WANTED}" ]]; then
- exec "${YETUS_HOME}/bin/${WANTED}" "${ARGV[@]}"
-fi
-
-#
-# this directory is ignored by git and maven
-#
-HADOOP_PATCHPROCESS=${HADOOP_PATCHPROCESS:-"${BINDIR}/../../patchprocess"}
-
-if [[ ! -d "${HADOOP_PATCHPROCESS}" ]]; then
- mkdir -p "${HADOOP_PATCHPROCESS}"
-fi
-
-mytmpdir=$(yetus_abs "${HADOOP_PATCHPROCESS}")
-ret=$?
-if [[ ${ret} != 0 ]]; then
- yetus_error "yetus-dl: Unable to cwd to ${HADOOP_PATCHPROCESS}"
- exit 1
-fi
-HADOOP_PATCHPROCESS=${mytmpdir}
-
-##
-## if we've already DL'd it, then short cut
-##
-if [[ -x "${HADOOP_PATCHPROCESS}/${YETUS_PREFIX}-${HADOOP_YETUS_VERSION}/bin/${WANTED}" ]]; then
- exec "${HADOOP_PATCHPROCESS}/${YETUS_PREFIX}-${HADOOP_YETUS_VERSION}/bin/${WANTED}" "${ARGV[@]}"
-fi
-
-##
-## need to DL, etc
-##
-
-BASEURL="https://archive.apache.org/dist/yetus/${HADOOP_YETUS_VERSION}/"
-TARBALL="${YETUS_PREFIX}-${HADOOP_YETUS_VERSION}-bin.tar"
-
-GPGBIN=$(command -v gpg)
-CURLBIN=$(command -v curl)
-
-if ! pushd "${HADOOP_PATCHPROCESS}" >/dev/null; then
- yetus_error "ERROR: yetus-dl: Cannot pushd to ${HADOOP_PATCHPROCESS}"
- exit 1
-fi
-
-if [[ -n "${CURLBIN}" ]]; then
- if ! "${CURLBIN}" -f -s -L -O "${BASEURL}/${TARBALL}.gz"; then
- yetus_error "ERROR: yetus-dl: unable to download ${BASEURL}/${TARBALL}.gz"
- exit 1
- fi
-else
- yetus_error "ERROR: yetus-dl requires curl."
- exit 1
-fi
-
-if [[ -n "${GPGBIN}" ]]; then
- if ! mkdir -p .gpg; then
- yetus_error "ERROR: yetus-dl: Unable to create ${HADOOP_PATCHPROCESS}/.gpg"
- exit 1
- fi
- if ! chmod -R 700 .gpg; then
- yetus_error "ERROR: yetus-dl: Unable to chmod ${HADOOP_PATCHPROCESS}/.gpg"
- exit 1
- fi
- if ! "${CURLBIN}" -s -L -o KEYS_YETUS https://dist.apache.org/repos/dist/release/yetus/KEYS; then
- yetus_error "ERROR: yetus-dl: unable to fetch https://dist.apache.org/repos/dist/release/yetus/KEYS"
- exit 1
- fi
- if ! "${CURLBIN}" -s -L -O "${BASEURL}/${TARBALL}.gz.asc"; then
- yetus_error "ERROR: yetus-dl: unable to fetch ${BASEURL}/${TARBALL}.gz.asc"
- exit 1
- fi
- if ! "${GPGBIN}" --homedir "${HADOOP_PATCHPROCESS}/.gpg" --import "${HADOOP_PATCHPROCESS}/KEYS_YETUS" >/dev/null 2>&1; then
- yetus_error "ERROR: yetus-dl: gpg unable to import ${HADOOP_PATCHPROCESS}/KEYS_YETUS"
- exit 1
- fi
- if ! "${GPGBIN}" --homedir "${HADOOP_PATCHPROCESS}/.gpg" --verify "${TARBALL}.gz.asc" >/dev/null 2>&1; then
- yetus_error "ERROR: yetus-dl: gpg verify of tarball in ${HADOOP_PATCHPROCESS} failed"
- exit 1
- fi
-fi
-
-if ! (gunzip -c "${TARBALL}.gz" | tar xpf -); then
- yetus_error "ERROR: ${TARBALL}.gz is corrupt. Investigate and then remove ${HADOOP_PATCHPROCESS} to try again."
- exit 1
-fi
-
-if [[ -x "${HADOOP_PATCHPROCESS}/${YETUS_PREFIX}-${HADOOP_YETUS_VERSION}/bin/${WANTED}" ]]; then
- popd >/dev/null
- exec "${HADOOP_PATCHPROCESS}/${YETUS_PREFIX}-${HADOOP_YETUS_VERSION}/bin/${WANTED}" "${ARGV[@]}"
-fi
-
-##
-## give up
-##
-yetus_error "ERROR: ${WANTED} is not part of Apache Yetus ${HADOOP_YETUS_VERSION}"
-exit 1
diff --git a/hadoop-hdds/client/pom.xml b/hadoop-hdds/client/pom.xml
index b8a91fc..de6f8bf 100644
--- a/hadoop-hdds/client/pom.xml
+++ b/hadoop-hdds/client/pom.xml
@@ -32,6 +32,11 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-test-utils</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-common</artifactId>
</dependency>
@@ -42,10 +47,8 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
+ <artifactId>hadoop-hdds-hadoop-dependency-test</artifactId>
<scope>test</scope>
- <type>test-jar</type>
</dependency>
</dependencies>
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
index 668fdaa..ad92621 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
@@ -18,10 +18,24 @@
package org.apache.hadoop.hdds.scm;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.security.cert.X509Certificate;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
import org.apache.hadoop.hdds.HddsUtils;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.function.SupplierWithIOException;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
@@ -41,7 +55,10 @@
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.util.Time;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import io.opentracing.Scope;
+import io.opentracing.Span;
import io.opentracing.util.GlobalTracer;
import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
import org.apache.ratis.thirdparty.io.grpc.Status;
@@ -52,28 +69,13 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.security.cert.X509Certificate;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
/**
* A Client for the storageContainer protocol for read object data.
*/
public class XceiverClientGrpc extends XceiverClientSpi {
static final Logger LOG = LoggerFactory.getLogger(XceiverClientGrpc.class);
private final Pipeline pipeline;
- private final Configuration config;
+ private final ConfigurationSource config;
private Map<UUID, XceiverClientProtocolServiceStub> asyncStubs;
private XceiverClientMetrics metrics;
private Map<UUID, ManagedChannel> channels;
@@ -94,7 +96,7 @@
* @param config -- Ozone Config
* @param caCert - SCM ca certificate.
*/
- public XceiverClientGrpc(Pipeline pipeline, Configuration config,
+ public XceiverClientGrpc(Pipeline pipeline, ConfigurationSource config,
X509Certificate caCert) {
super();
Preconditions.checkNotNull(pipeline);
@@ -121,7 +123,7 @@
* @param pipeline - Pipeline that defines the machines.
* @param config -- Ozone Config
*/
- public XceiverClientGrpc(Pipeline pipeline, Configuration config) {
+ public XceiverClientGrpc(Pipeline pipeline, ConfigurationSource config) {
this(pipeline, config, null);
}
@@ -265,14 +267,18 @@
private XceiverClientReply sendCommandWithTraceIDAndRetry(
ContainerCommandRequestProto request, List<CheckedBiFunction> validators)
throws IOException {
- try (Scope scope = GlobalTracer.get()
- .buildSpan("XceiverClientGrpc." + request.getCmdType().name())
- .startActive(true)) {
- ContainerCommandRequestProto finalPayload =
- ContainerCommandRequestProto.newBuilder(request)
- .setTraceID(TracingUtil.exportCurrentSpan()).build();
- return sendCommandWithRetry(finalPayload, validators);
- }
+
+ String spanName = "XceiverClientGrpc." + request.getCmdType().name();
+
+ return TracingUtil.executeInNewSpan(spanName,
+ (SupplierWithIOException<XceiverClientReply>) () -> {
+
+ ContainerCommandRequestProto finalPayload =
+ ContainerCommandRequestProto.newBuilder(request)
+ .setTraceID(TracingUtil.exportCurrentSpan()).build();
+ return sendCommandWithRetry(finalPayload, validators);
+
+ });
}
private XceiverClientReply sendCommandWithRetry(
@@ -387,9 +393,11 @@
public XceiverClientReply sendCommandAsync(
ContainerCommandRequestProto request)
throws IOException, ExecutionException, InterruptedException {
- try (Scope scope = GlobalTracer.get()
- .buildSpan("XceiverClientGrpc." + request.getCmdType().name())
- .startActive(true)) {
+
+ Span span = GlobalTracer.get()
+ .buildSpan("XceiverClientGrpc." + request.getCmdType().name()).start();
+
+ try (Scope scope = GlobalTracer.get().activateSpan(span)) {
ContainerCommandRequestProto finalPayload =
ContainerCommandRequestProto.newBuilder(request)
@@ -405,6 +413,9 @@
asyncReply.getResponse().get();
}
return asyncReply;
+
+ } finally {
+ span.finish();
}
}
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java
index 4ceff0b..8cc6e8d 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java
@@ -24,10 +24,11 @@
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.RemovalListener;
import com.google.common.cache.RemovalNotification;
-import org.apache.hadoop.conf.Configuration;
+
import org.apache.hadoop.hdds.conf.Config;
import org.apache.hadoop.hdds.conf.ConfigGroup;
import org.apache.hadoop.hdds.conf.ConfigType;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -52,6 +53,7 @@
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static org.apache.hadoop.hdds.conf.ConfigTag.OZONE;
import static org.apache.hadoop.hdds.conf.ConfigTag.PERFORMANCE;
+import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes.NO_REPLICA_FOUND;
/**
* XceiverClientManager is responsible for the lifecycle of XceiverClient
@@ -69,7 +71,7 @@
private static final Logger LOG =
LoggerFactory.getLogger(XceiverClientManager.class);
//TODO : change this to SCM configuration class
- private final Configuration conf;
+ private final ConfigurationSource conf;
private final Cache<String, XceiverClientSpi> clientCache;
private X509Certificate caCert;
@@ -83,12 +85,13 @@
*
* @param conf configuration
*/
- public XceiverClientManager(Configuration conf) throws IOException {
+ public XceiverClientManager(ConfigurationSource conf) throws IOException {
this(conf, OzoneConfiguration.of(conf).getObject(ScmClientConfig.class),
null);
}
- public XceiverClientManager(Configuration conf, ScmClientConfig clientConf,
+ public XceiverClientManager(ConfigurationSource conf,
+ ScmClientConfig clientConf,
String caCertPem) throws IOException {
Preconditions.checkNotNull(clientConf);
Preconditions.checkNotNull(conf);
@@ -166,7 +169,8 @@
throws IOException {
Preconditions.checkNotNull(pipeline);
Preconditions.checkArgument(pipeline.getNodes() != null);
- Preconditions.checkArgument(!pipeline.getNodes().isEmpty());
+ Preconditions.checkArgument(!pipeline.getNodes().isEmpty(),
+ NO_REPLICA_FOUND);
synchronized (clientCache) {
XceiverClientSpi info = getClient(pipeline, read);
@@ -292,6 +296,15 @@
}
/**
+ * Reset xceiver client metric.
+ */
+ public static synchronized void resetXceiverClientMetrics() {
+ if (metrics != null) {
+ metrics.reset();
+ }
+ }
+
+ /**
* Configuration for HDDS client.
*/
@ConfigGroup(prefix = "scm.container.client")
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java
index 7ca89ec..89a2592 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java
@@ -45,6 +45,10 @@
private MetricsRegistry registry;
public XceiverClientMetrics() {
+ init();
+ }
+
+ public void init() {
int numEnumEntries = ContainerProtos.Type.values().length;
this.registry = new MetricsRegistry(SOURCE_NAME);
@@ -92,7 +96,7 @@
containerOpsLatency[type.ordinal()].add(latencyNanos);
}
- public long getContainerOpsMetrics(ContainerProtos.Type type) {
+ public long getPendingContainerOpCountMetrics(ContainerProtos.Type type) {
return pendingOpsArray[type.ordinal()].value();
}
@@ -106,6 +110,11 @@
return opsArray[type.ordinal()].value();
}
+ @VisibleForTesting
+ public void reset() {
+ init();
+ }
+
public void unRegister() {
MetricsSystem ms = DefaultMetricsSystem.instance();
ms.unregisterSource(SOURCE_NAME);
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
index 0d12355..7ff5ab1 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
@@ -34,20 +34,24 @@
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsUtils;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.ratis.ContainerCommandRequestMessage;
+import org.apache.hadoop.hdds.ratis.RatisHelper;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.tracing.TracingUtil;
import org.apache.hadoop.util.Time;
-import org.apache.hadoop.hdds.ratis.RatisHelper;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Supplier;
import org.apache.ratis.client.RaftClient;
import org.apache.ratis.grpc.GrpcTlsConfig;
import org.apache.ratis.proto.RaftProtos;
@@ -61,12 +65,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-
-import io.opentracing.Scope;
-import io.opentracing.util.GlobalTracer;
-
/**
* An abstract implementation of {@link XceiverClientSpi} using Ratis.
* The underlying RPC mechanism can be chosen via the constructor.
@@ -77,13 +75,13 @@
public static XceiverClientRatis newXceiverClientRatis(
org.apache.hadoop.hdds.scm.pipeline.Pipeline pipeline,
- Configuration ozoneConf) {
+ ConfigurationSource ozoneConf) {
return newXceiverClientRatis(pipeline, ozoneConf, null);
}
public static XceiverClientRatis newXceiverClientRatis(
org.apache.hadoop.hdds.scm.pipeline.Pipeline pipeline,
- Configuration ozoneConf, X509Certificate caCert) {
+ ConfigurationSource ozoneConf, X509Certificate caCert) {
final String rpcType = ozoneConf
.get(ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY,
ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT);
@@ -100,7 +98,7 @@
private final AtomicReference<RaftClient> client = new AtomicReference<>();
private final RetryPolicy retryPolicy;
private final GrpcTlsConfig tlsConfig;
- private final Configuration ozoneConfiguration;
+ private final ConfigurationSource ozoneConfiguration;
// Map to track commit index at every server
private final ConcurrentHashMap<UUID, Long> commitInfoMap;
@@ -112,7 +110,7 @@
*/
private XceiverClientRatis(Pipeline pipeline, RpcType rpcType,
RetryPolicy retryPolicy, GrpcTlsConfig tlsConfig,
- Configuration configuration) {
+ ConfigurationSource configuration) {
super();
this.pipeline = pipeline;
this.rpcType = rpcType;
@@ -209,24 +207,27 @@
private CompletableFuture<RaftClientReply> sendRequestAsync(
ContainerCommandRequestProto request) {
- try (Scope scope = GlobalTracer.get()
- .buildSpan("XceiverClientRatis." + request.getCmdType().name())
- .startActive(true)) {
- final ContainerCommandRequestMessage message
- = ContainerCommandRequestMessage.toMessage(
+ return TracingUtil.executeInNewSpan(
+ "XceiverClientRatis." + request.getCmdType().name(),
+ (Supplier<CompletableFuture<RaftClientReply>>) () -> {
+ final ContainerCommandRequestMessage message
+ = ContainerCommandRequestMessage.toMessage(
request, TracingUtil.exportCurrentSpan());
- if (HddsUtils.isReadOnly(request)) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("sendCommandAsync ReadOnly {}", message);
+ if (HddsUtils.isReadOnly(request)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("sendCommandAsync ReadOnly {}", message);
+ }
+ return getClient().sendReadOnlyAsync(message);
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("sendCommandAsync {}", message);
+ }
+ return getClient().sendAsync(message);
+ }
+
}
- return getClient().sendReadOnlyAsync(message);
- } else {
- if (LOG.isDebugEnabled()) {
- LOG.debug("sendCommandAsync {}", message);
- }
- return getClient().sendAsync(message);
- }
- }
+
+ );
}
// gets the minimum log index replicated to all servers
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/HddsClientUtils.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/HddsClientUtils.java
index a87e13f..0f102c8 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/HddsClientUtils.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/HddsClientUtils.java
@@ -31,7 +31,7 @@
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.annotation.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.conf.RatisClientConfig;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
@@ -42,9 +42,6 @@
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
import org.apache.ratis.protocol.AlreadyClosedException;
import org.apache.ratis.protocol.GroupMismatchException;
import org.apache.ratis.protocol.NotReplicatedException;
@@ -216,56 +213,26 @@
* @param conf Configuration object
* @return list cache size
*/
- public static int getListCacheSize(Configuration conf) {
+ public static int getListCacheSize(ConfigurationSource conf) {
return conf.getInt(OzoneConfigKeys.OZONE_CLIENT_LIST_CACHE_SIZE,
OzoneConfigKeys.OZONE_CLIENT_LIST_CACHE_SIZE_DEFAULT);
}
/**
- * @return a default instance of {@link CloseableHttpClient}.
+ * Returns the s3VolumeName configured in ConfigurationSource.
+ * @param conf Configuration object
+ * @return s3 volume name
*/
- public static CloseableHttpClient newHttpClient() {
- return HddsClientUtils.newHttpClient(new Configuration());
- }
-
- /**
- * Returns a {@link CloseableHttpClient} configured by given configuration.
- * If conf is null, returns a default instance.
- *
- * @param conf configuration
- * @return a {@link CloseableHttpClient} instance.
- */
- public static CloseableHttpClient newHttpClient(Configuration conf) {
- long socketTimeout = OzoneConfigKeys
- .OZONE_CLIENT_SOCKET_TIMEOUT_DEFAULT;
- long connectionTimeout = OzoneConfigKeys
- .OZONE_CLIENT_CONNECTION_TIMEOUT_DEFAULT;
- if (conf != null) {
- socketTimeout = conf.getTimeDuration(
- OzoneConfigKeys.OZONE_CLIENT_SOCKET_TIMEOUT,
- OzoneConfigKeys.OZONE_CLIENT_SOCKET_TIMEOUT_DEFAULT,
- TimeUnit.MILLISECONDS);
- connectionTimeout = conf.getTimeDuration(
- OzoneConfigKeys.OZONE_CLIENT_CONNECTION_TIMEOUT,
- OzoneConfigKeys.OZONE_CLIENT_CONNECTION_TIMEOUT_DEFAULT,
- TimeUnit.MILLISECONDS);
- }
-
- CloseableHttpClient client = HttpClients.custom()
- .setDefaultRequestConfig(
- RequestConfig.custom()
- .setSocketTimeout(Math.toIntExact(socketTimeout))
- .setConnectTimeout(Math.toIntExact(connectionTimeout))
- .build())
- .build();
- return client;
+ public static String getS3VolumeName(ConfigurationSource conf) {
+ return conf.get(OzoneConfigKeys.OZONE_S3_VOLUME_NAME,
+ OzoneConfigKeys.OZONE_S3_VOLUME_NAME_DEFAULT);
}
/**
* Returns the maximum no of outstanding async requests to be handled by
* Standalone and Ratis client.
*/
- public static int getMaxOutstandingRequests(Configuration config) {
+ public static int getMaxOutstandingRequests(ConfigurationSource config) {
return OzoneConfiguration.of(config)
.getObject(RatisClientConfig.class)
.getMaxOutstandingRequests();
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockOutputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockOutputStream.java
index ffabbf3..f15a5e6 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockOutputStream.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockOutputStream.java
@@ -85,7 +85,9 @@
private final int bytesPerChecksum;
private int chunkIndex;
private final AtomicLong chunkOffset = new AtomicLong();
+ private final int streamBufferSize;
private final long streamBufferFlushSize;
+ private final boolean streamBufferFlushDelay;
private final long streamBufferMaxSize;
private final BufferPool bufferPool;
// The IOException will be set by response handling thread in case there is an
@@ -131,10 +133,10 @@
@SuppressWarnings("parameternumber")
public BlockOutputStream(BlockID blockID,
XceiverClientManager xceiverClientManager, Pipeline pipeline,
- long streamBufferFlushSize, long streamBufferMaxSize,
+ int streamBufferSize, long streamBufferFlushSize,
+ boolean streamBufferFlushDelay, long streamBufferMaxSize,
BufferPool bufferPool, ChecksumType checksumType,
- int bytesPerChecksum)
- throws IOException {
+ int bytesPerChecksum) throws IOException {
this.blockID = new AtomicReference<>(blockID);
KeyValue keyValue =
KeyValue.newBuilder().setKey("TYPE").setValue("KEY").build();
@@ -143,8 +145,10 @@
.addMetadata(keyValue);
this.xceiverClientManager = xceiverClientManager;
this.xceiverClient = xceiverClientManager.acquireClient(pipeline);
+ this.streamBufferSize = streamBufferSize;
this.streamBufferFlushSize = streamBufferFlushSize;
this.streamBufferMaxSize = streamBufferMaxSize;
+ this.streamBufferFlushDelay = streamBufferFlushDelay;
this.bufferPool = bufferPool;
this.bytesPerChecksum = bytesPerChecksum;
@@ -434,7 +438,9 @@
@Override
public void flush() throws IOException {
if (xceiverClientManager != null && xceiverClient != null
- && bufferPool != null && bufferPool.getSize() > 0) {
+ && bufferPool != null && bufferPool.getSize() > 0
+ && (!streamBufferFlushDelay ||
+ writtenDataLength - totalDataFlushedLength >= streamBufferSize)) {
try {
handleFlush(false);
} catch (InterruptedException | ExecutionException e) {
@@ -447,7 +453,6 @@
}
}
-
private void writeChunk(ChunkBuffer buffer)
throws IOException {
// This data in the buffer will be pushed to datanode and a reference will
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index 5fc48b0c..39a4d0a 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -28,28 +28,14 @@
<name>Apache Hadoop HDDS Common</name>
<packaging>jar</packaging>
- <properties>
- <hdds.version>0.6.0-SNAPSHOT</hdds.version>
- <log4j2.version>2.11.0</log4j2.version>
- <disruptor.version>3.4.2</disruptor.version>
- <declared.hdds.version>${hdds.version}</declared.hdds.version>
- </properties>
-
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop.version}</version>
+ <artifactId>hadoop-hdds-hadoop-dependency-client</artifactId>
</dependency>
<dependency>
<groupId>info.picocli</groupId>
<artifactId>picocli</artifactId>
- <version>3.9.6</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
@@ -63,14 +49,30 @@
</dependency>
<dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-config</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-test-utils</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>javax.annotation</groupId>
<artifactId>javax.annotation-api</artifactId>
- <version>1.2</version>
</dependency>
<dependency>
@@ -107,21 +109,9 @@
<dependency>
<groupId>com.google.errorprone</groupId>
<artifactId>error_prone_annotations</artifactId>
- <version>2.2.0</version>
<optional>true</optional>
</dependency>
- <dependency>
- <groupId>org.rocksdb</groupId>
- <artifactId>rocksdbjni</artifactId>
- <version>6.6.4</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
@@ -134,6 +124,16 @@
<version>${log4j2.version}</version>
</dependency>
<dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>com.lmax</groupId>
<artifactId>disruptor</artifactId>
<version>${disruptor.version}</version>
@@ -141,48 +141,39 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-pool2</artifactId>
- <version>2.6.0</version>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk15on</artifactId>
<version>${bouncycastle.version}</version>
</dependency>
- <!-- https://mvnrepository.com/artifact/commons-validator/commons-validator -->
<dependency>
<groupId>commons-validator</groupId>
<artifactId>commons-validator</artifactId>
- <version>1.6</version>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>io.jaegertracing</groupId>
<artifactId>jaeger-client</artifactId>
- <version>${jaeger.version}</version>
</dependency>
<dependency>
<groupId>io.opentracing</groupId>
<artifactId>opentracing-util</artifactId>
- <version>0.31.0</version>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
- <version>1.16</version>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.datatype</groupId>
- <artifactId>jackson-datatype-jsr310</artifactId>
- <version>${jackson2.version}</version>
- </dependency>
+
</dependencies>
@@ -212,36 +203,59 @@
</extensions>
<plugins>
<plugin>
+ <groupId>com.salesforce.servicelibs</groupId>
+ <artifactId>proto-backwards-compatibility</artifactId>
+ </plugin>
+ <plugin>
<groupId>org.xolstice.maven.plugins</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
<version>${protobuf-maven-plugin.version}</version>
<extensions>true</extensions>
- <configuration>
- <protocArtifact>
- com.google.protobuf:protoc:${datanode.protobuf-compile.version}:exe:${os.detected.classifier}
- </protocArtifact>
- <protoSourceRoot>${basedir}/src/main/proto/</protoSourceRoot>
- <includes>
- <include>DatanodeContainerProtocol.proto</include>
- </includes>
- <outputDirectory>target/generated-sources/java</outputDirectory>
- <clearOutputDirectory>false</clearOutputDirectory>
- </configuration>
<executions>
<execution>
- <id>compile-protoc</id>
- <goals>
- <goal>compile</goal>
- <goal>test-compile</goal>
- <goal>compile-custom</goal>
- <goal>test-compile-custom</goal>
- </goals>
- <configuration>
- <pluginId>grpc-java</pluginId>
- <pluginArtifact>
- io.grpc:protoc-gen-grpc-java:${datanode.grpc-compile.version}:exe:${os.detected.classifier}
- </pluginArtifact>
- </configuration>
+ <id>compile-protoc-3</id>
+ <goals>
+ <goal>compile</goal>
+ <goal>test-compile</goal>
+ <goal>compile-custom</goal>
+ <goal>test-compile-custom</goal>
+ </goals>
+ <configuration>
+ <protocArtifact>
+ com.google.protobuf:protoc:${protobuf-compile.version}:exe:${os.detected.classifier}
+ </protocArtifact>
+ <protoSourceRoot>${basedir}/src/main/proto/</protoSourceRoot>
+ <includes>
+ <include>DatanodeContainerProtocol.proto</include>
+ </includes>
+ <outputDirectory>target/generated-sources/java</outputDirectory>
+ <clearOutputDirectory>false</clearOutputDirectory>
+ <pluginId>grpc-java</pluginId>
+ <pluginArtifact>
+ io.grpc:protoc-gen-grpc-java:${grpc-compile.version}:exe:${os.detected.classifier}
+ </pluginArtifact>
+ </configuration>
+ </execution>
+ <execution>
+ <id>compile-protoc-2</id>
+ <goals>
+ <goal>compile</goal>
+ <goal>test-compile</goal>
+ </goals>
+ <configuration>
+ <protocArtifact>
+ com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}
+ </protocArtifact>
+ <protoSourceRoot>${basedir}/src/main/proto/</protoSourceRoot>
+ <includes>
+ <include>StorageContainerLocationProtocol.proto</include>
+ <include>hdds.proto</include>
+ <include>ScmBlockLocationProtocol.proto</include>
+ <include>SCMSecurityProtocol.proto</include>
+ </includes>
+ <outputDirectory>target/generated-sources/java</outputDirectory>
+ <clearOutputDirectory>false</clearOutputDirectory>
+ </configuration>
</execution>
</executions>
</plugin>
@@ -286,28 +300,6 @@
</source>
</configuration>
</execution>
- <execution>
- <id>compile-protoc</id>
- <goals>
- <goal>protoc</goal>
- </goals>
- <configuration>
- <protocVersion>${protobuf.version}</protocVersion>
- <protocCommand>${protoc.path}</protocCommand>
- <imports>
- <param>${basedir}/src/main/proto</param>
- </imports>
- <source>
- <directory>${basedir}/src/main/proto</directory>
- <includes>
- <include>StorageContainerLocationProtocol.proto</include>
- <include>hdds.proto</include>
- <include>ScmBlockLocationProtocol.proto</include>
- <include>SCMSecurityProtocol.proto</include>
- </includes>
- </source>
- </configuration>
- </execution>
</executions>
</plugin>
<plugin>
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index 3d2b703..9c77a1e 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -245,10 +245,20 @@
public static final String HDDS_DATANODE_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
public static final int HDDS_DATANODE_HTTP_BIND_PORT_DEFAULT = 9882;
public static final int HDDS_DATANODE_HTTPS_BIND_PORT_DEFAULT = 9883;
+
+ public static final String OZONE_DATANODE_HTTP_AUTH_CONFIG_PREFIX =
+ "hdds.datanode.http.auth.";
+ public static final String HDDS_DATANODE_HTTP_AUTH_TYPE =
+ OZONE_DATANODE_HTTP_AUTH_CONFIG_PREFIX + "type";
public static final String
HDDS_DATANODE_HTTP_KERBEROS_PRINCIPAL_KEY =
- "hdds.datanode.http.kerberos.principal";
+ OZONE_DATANODE_HTTP_AUTH_CONFIG_PREFIX + "kerberos.principal";
public static final String
HDDS_DATANODE_HTTP_KERBEROS_KEYTAB_FILE_KEY =
- "hdds.datanode.http.kerberos.keytab";
+ OZONE_DATANODE_HTTP_AUTH_CONFIG_PREFIX + "kerberos.keytab";
+
+ public static final String HDDS_DATANODE_RATIS_SERVER_REQUEST_TIMEOUT =
+ "hdds.datanode.ratis.server.request.timeout";
+ public static final String
+ HDDS_DATANODE_RATIS_SERVER_REQUEST_TIMEOUT_DEFAULT = "2m";
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
index acbd456..d1c2bc3 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
@@ -33,11 +33,11 @@
import java.util.OptionalInt;
import java.util.TimeZone;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.annotation.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -89,7 +89,8 @@
*
* @return Target {@code InetSocketAddress} for the SCM client endpoint.
*/
- public static InetSocketAddress getScmAddressForClients(Configuration conf) {
+ public static InetSocketAddress getScmAddressForClients(
+ ConfigurationSource conf) {
Optional<String> host = getHostNameFromConfigKeys(conf,
ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
@@ -116,7 +117,7 @@
* @throws IllegalArgumentException if configuration is not defined.
*/
public static InetSocketAddress getScmAddressForBlockClients(
- Configuration conf) {
+ ConfigurationSource conf) {
Optional<String> host = getHostNameFromConfigKeys(conf,
ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY,
ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
@@ -147,7 +148,8 @@
* @throws IllegalArgumentException if any values are not in the 'host'
* or host:port format.
*/
- public static Optional<String> getHostNameFromConfigKeys(Configuration conf,
+ public static Optional<String> getHostNameFromConfigKeys(
+ ConfigurationSource conf,
String... keys) {
for (final String key : keys) {
final String value = conf.getTrimmed(key);
@@ -206,7 +208,7 @@
* or host:port format.
*/
public static OptionalInt getPortNumberFromConfigKeys(
- Configuration conf, String... keys) {
+ ConfigurationSource conf, String... keys) {
for (final String key : keys) {
final String value = conf.getTrimmed(key);
final OptionalInt hostPort = getHostPort(value);
@@ -224,7 +226,7 @@
* @throws IllegalArgumentException If the configuration is invalid
*/
public static Collection<InetSocketAddress> getSCMAddresses(
- Configuration conf) {
+ ConfigurationSource conf) {
Collection<String> names =
conf.getTrimmedStringCollection(ScmConfigKeys.OZONE_SCM_NAMES);
if (names.isEmpty()) {
@@ -255,7 +257,7 @@
* @throws IllegalArgumentException If the configuration is invalid
*/
public static InetSocketAddress getReconAddresses(
- Configuration conf) {
+ ConfigurationSource conf) {
String name = conf.get(OZONE_RECON_ADDRESS_KEY);
if (StringUtils.isEmpty(name)) {
return null;
@@ -277,7 +279,8 @@
* @throws IllegalArgumentException if {@code conf} has more than one SCM
* address or it has none
*/
- public static InetSocketAddress getSingleSCMAddress(Configuration conf) {
+ public static InetSocketAddress getSingleSCMAddress(
+ ConfigurationSource conf) {
Collection<InetSocketAddress> singleton = getSCMAddresses(conf);
Preconditions.checkArgument(singleton.size() == 1,
MULTIPLE_SCM_NOT_YET_SUPPORTED);
@@ -295,7 +298,7 @@
* @throws UnknownHostException if the dfs.datanode.dns.interface
* option is used and the hostname can not be determined
*/
- public static String getHostName(Configuration conf)
+ public static String getHostName(ConfigurationSource conf)
throws UnknownHostException {
String name = conf.get(DFS_DATANODE_HOST_NAME_KEY);
if (name == null) {
@@ -498,7 +501,7 @@
* @param alias name of the credential to retreive
* @return String credential value or null
*/
- static String getPassword(Configuration conf, String alias) {
+ static String getPassword(ConfigurationSource conf, String alias) {
String password = null;
try {
char[] passchars = conf.getPassword(alias);
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/annotation/InterfaceStability.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/annotation/InterfaceStability.java
index 27e629a..4251344 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/annotation/InterfaceStability.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/annotation/InterfaceStability.java
@@ -21,9 +21,6 @@
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
-import org.apache.hadoop.hdds.annotation.InterfaceAudience.LimitedPrivate;
-import org.apache.hadoop.hdds.annotation.InterfaceAudience.Private;
-import org.apache.hadoop.hdds.annotation.InterfaceAudience.Public;
/**
* Annotation to inform users of how much to rely on a particular package,
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/HddsPrometheusConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/HddsPrometheusConfig.java
new file mode 100644
index 0000000..a95ad67
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/HddsPrometheusConfig.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.conf;
+
+/**
+ * The configuration class for the Prometheus endpoint.
+ */
+@ConfigGroup(prefix = "hdds.prometheus.")
+public class HddsPrometheusConfig {
+
+ @Config(key = "endpoint.token",
+ type = ConfigType.STRING,
+ defaultValue = "",
+ tags = { ConfigTag.SECURITY, ConfigTag.MANAGEMENT },
+ description = "Allowed authorization token while using prometheus " +
+ "servlet endpoint. This will disable SPNEGO based authentication on" +
+ " the endpoint."
+ )
+ private String prometheusEndpointToken;
+
+ public String getPrometheusEndpointToken() {
+ return prometheusEndpointToken;
+ }
+
+ public void setPrometheusEndpointToken(String prometheusEndpointToken) {
+ this.prometheusEndpointToken = prometheusEndpointToken;
+ }
+}
\ No newline at end of file
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
index fda3c86..65c2802 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,7 +18,6 @@
package org.apache.hadoop.hdds.conf;
-import javax.annotation.PostConstruct;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
@@ -27,30 +26,44 @@
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import java.io.IOException;
-import java.lang.reflect.Field;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
import java.net.URL;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.stream.Collectors;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.annotation.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
/**
* Configuration for ozone.
*/
@InterfaceAudience.Private
-public class OzoneConfiguration extends Configuration {
+public class OzoneConfiguration extends Configuration
+ implements ConfigurationSource {
static {
activate();
}
+ public static OzoneConfiguration of(ConfigurationSource source) {
+ if (source instanceof LegacyHadoopConfigurationSource) {
+ return new OzoneConfiguration(((LegacyHadoopConfigurationSource) source)
+ .getOriginalHadoopConfiguration());
+ }
+ return (OzoneConfiguration) source;
+ }
+
+ public static OzoneConfiguration of(OzoneConfiguration source) {
+ return source;
+ }
+
public static OzoneConfiguration of(Configuration conf) {
Preconditions.checkNotNull(conf);
@@ -99,157 +112,6 @@
}
/**
- * Create a Configuration object and inject the required configuration values.
- *
- * @param configurationClass The class where the fields are annotated with
- * the configuration.
- * @return Initiated java object where the config fields are injected.
- */
- public <T> T getObject(Class<T> configurationClass) {
-
- T configObject;
-
- try {
- configObject = configurationClass.newInstance();
- } catch (InstantiationException | IllegalAccessException e) {
- throw new ConfigurationException(
- "Configuration class can't be created: " + configurationClass, e);
- }
- ConfigGroup configGroup =
- configurationClass.getAnnotation(ConfigGroup.class);
-
- String prefix = configGroup.prefix();
-
- injectConfiguration(configurationClass, configObject, prefix);
-
- callPostConstruct(configurationClass, configObject);
-
- return configObject;
-
- }
-
- private <T> void injectConfiguration(Class<T> configurationClass,
- T configObject, String prefix) {
- injectConfigurationToObject(configurationClass, configObject, prefix);
- Class<? super T> superClass = configurationClass.getSuperclass();
- while (superClass != null) {
- injectConfigurationToObject(superClass, configObject, prefix);
- superClass = superClass.getSuperclass();
- }
- }
-
- private <T> void callPostConstruct(Class<T> configurationClass,
- T configObject) {
- for (Method method : configurationClass.getMethods()) {
- if (method.isAnnotationPresent(PostConstruct.class)) {
- try {
- method.invoke(configObject);
- } catch (IllegalAccessException ex) {
- throw new IllegalArgumentException(
- "@PostConstruct method in " + configurationClass
- + " is not accessible");
- } catch (InvocationTargetException e) {
- if (e.getCause() instanceof RuntimeException) {
- throw (RuntimeException) e.getCause();
- } else {
- throw new IllegalArgumentException(
- "@PostConstruct can't be executed on " + configurationClass
- + " after configObject "
- + "injection", e);
- }
- }
- }
- }
- }
-
- private <T> void injectConfigurationToObject(Class<T> configurationClass,
- T configuration, String prefix) {
- for (Field field : configurationClass.getDeclaredFields()) {
- if (field.isAnnotationPresent(Config.class)) {
-
- String fieldLocation =
- configurationClass + "." + field.getName();
-
- Config configAnnotation = field.getAnnotation(Config.class);
-
- String key = prefix + "." + configAnnotation.key();
-
- ConfigType type = configAnnotation.type();
-
- if (type == ConfigType.AUTO) {
- type = detectConfigType(field.getType(), fieldLocation);
- }
-
- //Note: default value is handled by ozone-default.xml. Here we can
- //use any default.
- try {
- switch (type) {
- case STRING:
- forcedFieldSet(field, configuration, get(key));
- break;
- case INT:
- forcedFieldSet(field, configuration, getInt(key, 0));
- break;
- case BOOLEAN:
- forcedFieldSet(field, configuration, getBoolean(key, false));
- break;
- case LONG:
- forcedFieldSet(field, configuration, getLong(key, 0));
- break;
- case TIME:
- forcedFieldSet(field, configuration,
- getTimeDuration(key, 0, configAnnotation.timeUnit()));
- break;
- default:
- throw new ConfigurationException(
- "Unsupported ConfigType " + type + " on " + fieldLocation);
- }
- } catch (IllegalAccessException e) {
- throw new ConfigurationException(
- "Can't inject configuration to " + fieldLocation, e);
- }
-
- }
- }
- }
-
- /**
- * Set the value of one field even if it's private.
- */
- private <T> void forcedFieldSet(Field field, T object, Object value)
- throws IllegalAccessException {
- boolean accessChanged = false;
- if (!field.isAccessible()) {
- field.setAccessible(true);
- accessChanged = true;
- }
- field.set(object, value);
- if (accessChanged) {
- field.setAccessible(false);
- }
- }
-
- private ConfigType detectConfigType(Class<?> parameterType,
- String methodLocation) {
- ConfigType type;
- if (parameterType == String.class) {
- type = ConfigType.STRING;
- } else if (parameterType == Integer.class || parameterType == int.class) {
- type = ConfigType.INT;
- } else if (parameterType == Long.class || parameterType == long.class) {
- type = ConfigType.LONG;
- } else if (parameterType == Boolean.class
- || parameterType == boolean.class) {
- type = ConfigType.BOOLEAN;
- } else {
- throw new ConfigurationException(
- "Unsupported configuration type " + parameterType + " in "
- + methodLocation);
- }
- return type;
- }
-
- /**
* Class to marshall/un-marshall configuration from xml files.
*/
@XmlAccessorType(XmlAccessType.FIELD)
@@ -380,6 +242,14 @@
}
@Override
+ public Collection<String> getConfigKeys() {
+ return getProps().keySet()
+ .stream()
+ .map(Object::toString)
+ .collect(Collectors.toList());
+ }
+
+ @Override
public Map<String, String> getPropsWithPrefix(String confPrefix) {
Properties props = getProps();
Map<String, String> configMap = new HashMap<>();
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/DUFactory.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/DUFactory.java
index 118da1f5..5b57b34 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/DUFactory.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/DUFactory.java
@@ -17,16 +17,16 @@
*/
package org.apache.hadoop.hdds.fs;
-import org.apache.hadoop.conf.Configuration;
+import java.io.File;
+import java.time.Duration;
+
import org.apache.hadoop.hdds.conf.Config;
import org.apache.hadoop.hdds.conf.ConfigGroup;
import org.apache.hadoop.hdds.conf.ConfigTag;
import org.apache.hadoop.hdds.conf.ConfigType;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import java.io.File;
-import java.time.Duration;
-
/**
* Uses DU for all volumes. Saves used value in cache file.
*/
@@ -40,7 +40,8 @@
private Conf conf;
@Override
- public SpaceUsageCheckFactory setConfiguration(Configuration configuration) {
+ public SpaceUsageCheckFactory setConfiguration(
+ ConfigurationSource configuration) {
conf = OzoneConfiguration.of(configuration).getObject(Conf.class);
return this;
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/DedicatedDiskSpaceUsageFactory.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/DedicatedDiskSpaceUsageFactory.java
index 37953d9..3ed74c9 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/DedicatedDiskSpaceUsageFactory.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/DedicatedDiskSpaceUsageFactory.java
@@ -17,16 +17,16 @@
*/
package org.apache.hadoop.hdds.fs;
-import org.apache.hadoop.conf.Configuration;
+import java.io.File;
+import java.time.Duration;
+
import org.apache.hadoop.hdds.conf.Config;
import org.apache.hadoop.hdds.conf.ConfigGroup;
import org.apache.hadoop.hdds.conf.ConfigTag;
import org.apache.hadoop.hdds.conf.ConfigType;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import java.io.File;
-import java.time.Duration;
-
/**
* Uses DedicatedDiskSpaceUsage for all volumes. Does not save results since
* the information is relatively cheap to obtain.
@@ -38,7 +38,8 @@
private Conf conf;
@Override
- public SpaceUsageCheckFactory setConfiguration(Configuration configuration) {
+ public SpaceUsageCheckFactory setConfiguration(
+ ConfigurationSource configuration) {
conf = OzoneConfiguration.of(configuration).getObject(Conf.class);
return this;
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/SpaceUsageCheckFactory.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/SpaceUsageCheckFactory.java
index 3a3960b..0205de5 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/SpaceUsageCheckFactory.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/fs/SpaceUsageCheckFactory.java
@@ -17,21 +17,22 @@
*/
package org.apache.hadoop.hdds.fs;
-import org.apache.hadoop.hdds.annotation.InterfaceAudience;
-import org.apache.hadoop.hdds.annotation.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.conf.Config;
-import org.apache.hadoop.hdds.conf.ConfigGroup;
-import org.apache.hadoop.hdds.conf.ConfigTag;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.File;
import java.io.UncheckedIOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
+import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.annotation.InterfaceStability;
+import org.apache.hadoop.hdds.conf.Config;
+import org.apache.hadoop.hdds.conf.ConfigGroup;
+import org.apache.hadoop.hdds.conf.ConfigTag;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* Configures disk space checks (du, df, etc.) for HDDS volumes, allowing
* different implementations and parameters for different volumes.
@@ -57,7 +58,7 @@
* Updates the factory with global configuration.
* @return factory configured with {@code conf}
*/
- default SpaceUsageCheckFactory setConfiguration(Configuration conf) {
+ default SpaceUsageCheckFactory setConfiguration(ConfigurationSource conf) {
// override if configurable
return this;
}
@@ -68,14 +69,16 @@
* Defaults to {@link DUFactory} if no class is configured or it cannot be
* instantiated.
*/
- static SpaceUsageCheckFactory create(Configuration config) {
+ static SpaceUsageCheckFactory create(ConfigurationSource config) {
Conf conf = OzoneConfiguration.of(config).getObject(Conf.class);
Class<? extends SpaceUsageCheckFactory> aClass = null;
String className = conf.getClassName();
if (className != null && !className.isEmpty()) {
try {
- aClass = config.getClassByName(className)
- .asSubclass(SpaceUsageCheckFactory.class);
+ aClass =
+ SpaceUsageCheckFactory.class
+ .getClassLoader().loadClass(className)
+ .asSubclass(SpaceUsageCheckFactory.class);
} catch (ClassNotFoundException | RuntimeException e) {
Logger log = LoggerFactory.getLogger(SpaceUsageCheckFactory.class);
log.warn("Error trying to create SpaceUsageCheckFactory: '{}'",
@@ -91,7 +94,8 @@
aClass.getConstructor();
instance = constructor.newInstance();
} catch (IllegalAccessException | InstantiationException |
- InvocationTargetException | NoSuchMethodException e) {
+ InvocationTargetException | NoSuchMethodException |
+ ClassCastException e) {
Logger log = LoggerFactory.getLogger(SpaceUsageCheckFactory.class);
log.warn("Error trying to create {}", aClass, e);
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/function/SupplierWithIOException.java
similarity index 70%
copy from hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/package-info.java
copy to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/function/SupplierWithIOException.java
index 80c1985..a30e69a 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/package-info.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/function/SupplierWithIOException.java
@@ -15,7 +15,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.ozone.web.ozShell;
+package org.apache.hadoop.hdds.function;
+
+import java.io.IOException;
+
/**
- * Tests for ozone shell..
+ * Functional interface like java.util.function.Supplier but with
+ * checked exception.
*/
+@FunctionalInterface
+public interface SupplierWithIOException<T> {
+
+ /**
+ * Return the given output..
+ *
+ * @return the function result
+ */
+ T get() throws IOException;
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
index 0863b9c..08ead55 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
@@ -28,7 +28,7 @@
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
@@ -51,7 +51,6 @@
import org.apache.ratis.retry.RetryPolicy;
import org.apache.ratis.rpc.RpcType;
import org.apache.ratis.rpc.SupportedRpcType;
-import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.apache.ratis.util.TimeDuration;
import org.slf4j.Logger;
@@ -65,8 +64,9 @@
// Prefix for Ratis Server GRPC and Ratis client conf.
String HDDS_DATANODE_RATIS_PREFIX_KEY = "hdds.ratis.";
+ String RAFT_SERVER_PREFIX_KEY = "raft.server";
String HDDS_DATANODE_RATIS_SERVER_PREFIX_KEY =
- HDDS_DATANODE_RATIS_PREFIX_KEY + RaftServerConfigKeys.PREFIX;
+ HDDS_DATANODE_RATIS_PREFIX_KEY + RAFT_SERVER_PREFIX_KEY;
String HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY =
HDDS_DATANODE_RATIS_PREFIX_KEY + RaftClientConfigKeys.PREFIX;
String HDDS_DATANODE_RATIS_GRPC_PREFIX_KEY =
@@ -146,27 +146,27 @@
static RaftClient newRaftClient(RpcType rpcType, Pipeline pipeline,
RetryPolicy retryPolicy, GrpcTlsConfig tlsConfig,
- Configuration ozoneConfiguration) throws IOException {
+ ConfigurationSource ozoneConfiguration) throws IOException {
return newRaftClient(rpcType,
toRaftPeerId(pipeline.getLeaderNode()),
newRaftGroup(RaftGroupId.valueOf(pipeline.getId().getId()),
pipeline.getNodes()), retryPolicy, tlsConfig, ozoneConfiguration);
}
- static RpcType getRpcType(Configuration conf) {
+ static RpcType getRpcType(ConfigurationSource conf) {
return SupportedRpcType.valueOfIgnoreCase(conf.get(
ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY,
ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT));
}
- static RaftClient newRaftClient(RaftPeer leader, Configuration conf) {
+ static RaftClient newRaftClient(RaftPeer leader, ConfigurationSource conf) {
return newRaftClient(getRpcType(conf), leader,
RatisHelper.createRetryPolicy(conf), conf);
}
static RaftClient newRaftClient(RpcType rpcType, RaftPeer leader,
RetryPolicy retryPolicy, GrpcTlsConfig tlsConfig,
- Configuration configuration) {
+ ConfigurationSource configuration) {
return newRaftClient(rpcType, leader.getId(),
newRaftGroup(Collections.singletonList(leader)), retryPolicy,
tlsConfig, configuration);
@@ -174,7 +174,7 @@
static RaftClient newRaftClient(RpcType rpcType, RaftPeer leader,
RetryPolicy retryPolicy,
- Configuration ozoneConfiguration) {
+ ConfigurationSource ozoneConfiguration) {
return newRaftClient(rpcType, leader.getId(),
newRaftGroup(Collections.singletonList(leader)), retryPolicy, null,
ozoneConfiguration);
@@ -183,7 +183,7 @@
@SuppressWarnings("checkstyle:ParameterNumber")
static RaftClient newRaftClient(RpcType rpcType, RaftPeerId leader,
RaftGroup group, RetryPolicy retryPolicy,
- GrpcTlsConfig tlsConfig, Configuration ozoneConfiguration) {
+ GrpcTlsConfig tlsConfig, ConfigurationSource ozoneConfiguration) {
if (LOG.isTraceEnabled()) {
LOG.trace("newRaftClient: {}, leader={}, group={}",
rpcType, leader, group);
@@ -215,14 +215,14 @@
* @param ozoneConf
* @param raftProperties
*/
- static void createRaftClientProperties(Configuration ozoneConf,
+ static void createRaftClientProperties(ConfigurationSource ozoneConf,
RaftProperties raftProperties) {
// As for client we do not require server and grpc server/tls. exclude them.
Map<String, String> ratisClientConf =
ozoneConf.getPropsWithPrefix(HDDS_DATANODE_RATIS_PREFIX_KEY);
ratisClientConf.forEach((key, val) -> {
- if (!(key.startsWith(RaftServerConfigKeys.PREFIX) ||
+ if (!(key.startsWith(RAFT_SERVER_PREFIX_KEY) ||
key.startsWith(GrpcConfigKeys.TLS.PREFIX) ||
key.startsWith(GrpcConfigKeys.Server.PREFIX))) {
raftProperties.set(key, val);
@@ -238,7 +238,7 @@
* @param ozoneConf
* @param raftProperties
*/
- static void createRaftServerProperties(Configuration ozoneConf,
+ static void createRaftServerProperties(ConfigurationSource ozoneConf,
RaftProperties raftProperties) {
Map<String, String> ratisServerConf =
@@ -253,7 +253,7 @@
static Map<String, String> getDatanodeRatisPrefixProps(
- Configuration configuration) {
+ ConfigurationSource configuration) {
return configuration.getPropsWithPrefix(HDDS_DATANODE_RATIS_PREFIX_KEY);
}
@@ -269,11 +269,7 @@
return tlsConfig;
}
-
-
-
-
- static RetryPolicy createRetryPolicy(Configuration conf) {
+ static RetryPolicy createRetryPolicy(ConfigurationSource conf) {
int maxRetryCount =
conf.getInt(OzoneConfigKeys.DFS_RATIS_CLIENT_REQUEST_MAX_RETRIES_KEY,
OzoneConfigKeys.
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ByteStringConversion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ByteStringConversion.java
index c51d66a..dc44392 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ByteStringConversion.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ByteStringConversion.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.hdds.scm;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations;
@@ -44,7 +44,7 @@
* @see ByteBuffer
*/
public static Function<ByteBuffer, ByteString> createByteBufferConversion(
- Configuration config){
+ ConfigurationSource config){
boolean unsafeEnabled =
config!=null && config.getBoolean(
OzoneConfigKeys.OZONE_UNSAFEBYTEOPERATIONS_ENABLED,
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ContainerPlacementStatus.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ContainerPlacementStatus.java
new file mode 100644
index 0000000..8f3334f
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ContainerPlacementStatus.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.hdds.scm;
+
+/**
+ * Interface to allow container placement status to be queried to ensure a
+ * container meets its placement policy (number of racks etc).
+ */
+public interface ContainerPlacementStatus {
+
+ /**
+ * Returns a boolean indicating if the container replicas meet the desired
+ * placement policy. That is, they are placed on a sufficient number of
+ * racks, or node groups etc. This does not check if the container is over
+ * or under replicated, as it is possible for a container to have enough
+ * replicas and still not meet the placement rules.
+ * @return True if the containers meet the policy. False otherwise.
+ */
+ boolean isPolicySatisfied();
+
+ /**
+ * Returns an String describing why a container does not meet the placement
+ * policy.
+ * @return String indicating the reason the policy is not satisfied or null if
+ * the policy is satisfied.
+ */
+ String misReplicatedReason();
+
+ /**
+ * If the container does not meet the placement policy, return an integer
+ * indicating how many additional replicas are required so the container
+ * meets the placement policy. Otherwise return zero.
+ * Note the count returned are the number of replicas needed to meet the
+ * placement policy. The container may need additional replicas if it is
+ * under replicated. The container could also have sufficient replicas but
+ * require more to make it meet the policy, as the existing replicas are not
+ * placed correctly.
+ * @return The number of additional replicas required, or zero.
+ */
+ int misReplicationCount();
+
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java
index f6a0e8b..46671df 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java
@@ -42,4 +42,14 @@
List<DatanodeDetails> chooseDatanodes(List<DatanodeDetails> excludedNodes,
List<DatanodeDetails> favoredNodes, int nodesRequired, long sizeRequired)
throws IOException;
+
+ /**
+ * Given a list of datanode and the number of replicas required, return
+ * a PlacementPolicyStatus object indicating if the container meets the
+ * placement policy - ie is it on the correct number of racks, etc.
+ * @param dns List of datanodes holding a replica of the container
+ * @param replicas The expected number of replicas
+ */
+ ContainerPlacementStatus validateContainerPlacement(
+ List<DatanodeDetails> dns, int replicas);
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java
index c98546b..73701ea 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java
@@ -45,6 +45,19 @@
)
private String keytab;
+ @Config(key = "unknown-container.action",
+ type = ConfigType.STRING,
+ defaultValue = "WARN",
+ tags = { ConfigTag.SCM, ConfigTag.MANAGEMENT },
+ description =
+ "The action taken by SCM to process unknown "
+ + "containers that reported by Datanodes. The default "
+ + "action is just logging container not found warning, "
+ + "another available action is DELETE action. "
+ + "These unknown containers will be deleted under this "
+ + "action way."
+ )
+ private String action;
public void setKerberosPrincipal(String kerberosPrincipal) {
this.principal = kerberosPrincipal;
@@ -55,6 +68,10 @@
this.keytab = kerberosKeytab;
}
+ public void setUnknownContainerAction(String unknownContainerAction) {
+ this.action = unknownContainerAction;
+ }
+
public String getKerberosPrincipal() {
return this.principal;
}
@@ -63,6 +80,10 @@
return this.keytab;
}
+ public String getUnknownContainerAction() {
+ return this.action;
+ }
+
/**
* Configuration strings class.
* required for SCMSecurityProtocol where the KerberosInfo references
@@ -77,4 +98,7 @@
public static final String HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY =
"hdds.scm.kerberos.keytab.file";
}
+
+ public static final String HDDS_SCM_UNKNOWN_CONTAINER_ACTION =
+ "hdds.scm.unknown-container.action";
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 4eda6f8..a028031 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -47,7 +47,7 @@
public static final String DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_KEY
= "dfs.container.ratis.num.write.chunk.threads";
public static final int DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT
- = 60;
+ = 10;
public static final String DFS_CONTAINER_RATIS_REPLICATION_LEVEL_KEY
= "dfs.container.ratis.replication.level";
public static final ReplicationLevel
@@ -361,7 +361,7 @@
"network-topology-default.xml";
public static final String HDDS_TRACING_ENABLED = "hdds.tracing.enabled";
- public static final boolean HDDS_TRACING_ENABLED_DEFAULT = true;
+ public static final boolean HDDS_TRACING_ENABLED_DEFAULT = false;
/**
* Never constructed.
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java
index 5c58e92..a599021 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java
@@ -17,45 +17,35 @@
*/
package org.apache.hadoop.hdds.scm.container;
-import static java.lang.Math.max;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.PropertyAccessor;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.google.common.base.Preconditions;
import java.io.Externalizable;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
+import java.time.Instant;
import java.util.Arrays;
import java.util.Comparator;
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
+
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.util.Time;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.google.common.base.Preconditions;
+import static java.lang.Math.max;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+
/**
* Class wraps ozone container info.
*/
public class ContainerInfo implements Comparator<ContainerInfo>,
Comparable<ContainerInfo>, Externalizable {
- private static final ObjectWriter WRITER;
private static final String SERIALIZATION_ERROR_MSG = "Java serialization not"
+ " supported. Use protobuf instead.";
- static {
- ObjectMapper mapper = new ObjectMapper();
- mapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
- mapper
- .setVisibility(PropertyAccessor.GETTER, JsonAutoDetect.Visibility.NONE);
- WRITER = mapper.writerWithDefaultPrettyPrinter();
- }
private HddsProtos.LifeCycleState state;
@JsonIgnore
@@ -64,9 +54,9 @@
private ReplicationType replicationType;
private long usedBytes;
private long numberOfKeys;
- private long lastUsed;
+ private Instant lastUsed;
// The wall-clock ms since the epoch at which the current state enters.
- private long stateEnterTime;
+ private Instant stateEnterTime;
private String owner;
private long containerID;
private long deleteTransactionId;
@@ -98,9 +88,9 @@
this.pipelineID = pipelineID;
this.usedBytes = usedBytes;
this.numberOfKeys = numberOfKeys;
- this.lastUsed = Time.monotonicNow();
+ this.lastUsed = Instant.ofEpochMilli(Time.now());
this.state = state;
- this.stateEnterTime = stateEnterTime;
+ this.stateEnterTime = Instant.ofEpochMilli(stateEnterTime);
this.owner = owner;
this.deleteTransactionId = deleteTransactionId;
this.sequenceId = sequenceId;
@@ -142,7 +132,7 @@
this.state = state;
}
- public long getStateEnterTime() {
+ public Instant getStateEnterTime() {
return stateEnterTime;
}
@@ -196,7 +186,7 @@
*
* @return time in milliseconds.
*/
- public long getLastUsed() {
+ public Instant getLastUsed() {
return lastUsed;
}
@@ -205,7 +195,7 @@
}
public void updateLastUsedTime() {
- lastUsed = Time.monotonicNow();
+ lastUsed = Instant.ofEpochMilli(Time.now());
}
public HddsProtos.ContainerInfoProto getProtobuf() {
@@ -215,7 +205,8 @@
return builder.setContainerID(getContainerID())
.setUsedBytes(getUsedBytes())
.setNumberOfKeys(getNumberOfKeys()).setState(getState())
- .setStateEnterTime(getStateEnterTime()).setContainerID(getContainerID())
+ .setStateEnterTime(getStateEnterTime().toEpochMilli())
+ .setContainerID(getContainerID())
.setDeleteTransactionId(getDeleteTransactionId())
.setPipelineID(getPipelineID().getProtobuf())
.setReplicationFactor(getReplicationFactor())
@@ -292,7 +283,8 @@
*/
@Override
public int compare(ContainerInfo o1, ContainerInfo o2) {
- return Long.compare(o1.getLastUsed(), o2.getLastUsed());
+ return Long.compare(
+ o1.getLastUsed().toEpochMilli(), o2.getLastUsed().toEpochMilli());
}
/**
@@ -312,15 +304,7 @@
return this.compare(this, o);
}
- /**
- * Returns a JSON string of this object.
- *
- * @return String - json string
- * @throws IOException
- */
- public String toJsonString() throws IOException {
- return WRITER.writeValueAsString(this);
- }
+
/**
* Returns private data that is set on this containerInfo.
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetworkTopologyImpl.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetworkTopologyImpl.java
index 58394f1..1db46c4 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetworkTopologyImpl.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetworkTopologyImpl.java
@@ -34,7 +34,7 @@
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT;
import static org.apache.hadoop.hdds.scm.net.NetConstants.SCOPE_REVERSE_STR;
import static org.apache.hadoop.hdds.scm.net.NetConstants.ANCESTOR_GENERATION_DEFAULT;
@@ -60,7 +60,7 @@
/** Lock to coordinate cluster tree access. */
private ReadWriteLock netlock = new ReentrantReadWriteLock(true);
- public NetworkTopologyImpl(Configuration conf) {
+ public NetworkTopologyImpl(ConfigurationSource conf) {
schemaManager = NodeSchemaManager.getInstance();
schemaManager.init(conf);
maxLevel = schemaManager.getMaxLevel();
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NodeSchemaManager.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NodeSchemaManager.java
index c60c2c8..698b9da 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NodeSchemaManager.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NodeSchemaManager.java
@@ -24,7 +24,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import java.util.ArrayList;
import java.util.Arrays;
@@ -55,7 +55,7 @@
return instance;
}
- public void init(Configuration conf) {
+ public void init(ConfigurationSource conf) {
/**
* Load schemas from network topology schema configuration file
*/
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
index 3ec3277..dff739a 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
@@ -77,6 +77,18 @@
throws IOException;
/**
+ * Ask SCM the location of a batch of containers. SCM responds with a group of
+ * nodes where these containers and their replicas are located.
+ *
+ * @param containerIDs - IDs of a batch of containers.
+ * @return List of ContainerWithPipeline
+ * - the container info with the pipeline.
+ * @throws IOException
+ */
+ List<ContainerWithPipeline> getContainerWithPipelineBatch(
+ List<Long> containerIDs) throws IOException;
+
+ /**
* Ask SCM a list of containers with a range of container names
* and the limit of count.
* Search container names between start name(exclusive), and
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
index dffae11..b0040cf 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
@@ -35,6 +35,7 @@
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ForceExitSafeModeResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerWithPipelineRequestProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerWithPipelineBatchRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetPipelineRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetPipelineResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.InSafeModeRequestProto;
@@ -197,6 +198,40 @@
/**
* {@inheritDoc}
*/
+ public List<ContainerWithPipeline> getContainerWithPipelineBatch(
+ List<Long> containerIDs) throws IOException {
+ for (Long containerID: containerIDs) {
+ Preconditions.checkState(containerID >= 0,
+ "Container ID cannot be negative");
+ }
+
+ GetContainerWithPipelineBatchRequestProto request =
+ GetContainerWithPipelineBatchRequestProto.newBuilder()
+ .setTraceID(TracingUtil.exportCurrentSpan())
+ .addAllContainerIDs(containerIDs)
+ .build();
+
+ ScmContainerLocationResponse response =
+ submitRequest(Type.GetContainerWithPipelineBatch,
+ (builder) -> builder
+ .setGetContainerWithPipelineBatchRequest(request));
+
+ List<HddsProtos.ContainerWithPipeline> protoCps = response
+ .getGetContainerWithPipelineBatchResponse()
+ .getContainerWithPipelinesList();
+
+ List<ContainerWithPipeline> cps = new ArrayList<>();
+
+ for (HddsProtos.ContainerWithPipeline cp : protoCps) {
+ cps.add(ContainerWithPipeline.fromProtobuf(cp));
+ }
+
+ return cps;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
@Override
public List<ContainerInfo> listContainer(long startContainerID, int count)
throws IOException {
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
index 9d077f6..394a0c3 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
@@ -19,14 +19,6 @@
package org.apache.hadoop.hdds.security.x509;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.ratis.thirdparty.io.netty.handler.ssl.SslProvider;
-import org.bouncycastle.jce.provider.BouncyCastleProvider;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.Provider;
@@ -34,6 +26,10 @@
import java.time.Duration;
import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+
+import com.google.common.base.Preconditions;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED_DEFAULT;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_ALGORITHM;
@@ -71,6 +67,10 @@
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
+import org.apache.ratis.thirdparty.io.netty.handler.ssl.SslProvider;
+import org.bouncycastle.jce.provider.BouncyCastleProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A class that deals with all Security related configs in HDDS.
@@ -82,7 +82,7 @@
private static final Logger LOG =
LoggerFactory.getLogger(SecurityConfig.class);
private static volatile Provider provider;
- private final Configuration configuration;
+ private final ConfigurationSource configuration;
private final int size;
private final String keyAlgo;
private final String providerString;
@@ -106,7 +106,7 @@
*
* @param configuration - HDDS Configuration
*/
- public SecurityConfig(Configuration configuration) {
+ public SecurityConfig(ConfigurationSource configuration) {
Preconditions.checkNotNull(configuration, "Configuration cannot be null");
this.configuration = configuration;
this.size = this.configuration.getInt(HDDS_KEY_LEN, HDDS_DEFAULT_KEY_LEN);
@@ -305,7 +305,7 @@
*
* @return Configuration
*/
- public Configuration getConfiguration() {
+ public ConfigurationSource getConfiguration() {
return configuration;
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/GrpcServerInterceptor.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/GrpcServerInterceptor.java
index b63af12..bd35d56 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/GrpcServerInterceptor.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/GrpcServerInterceptor.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.hdds.tracing;
import io.opentracing.Scope;
+import io.opentracing.Span;
+import io.opentracing.util.GlobalTracer;
import org.apache.ratis.thirdparty.io.grpc.ForwardingServerCallListener.SimpleForwardingServerCallListener;
import org.apache.ratis.thirdparty.io.grpc.Metadata;
import org.apache.ratis.thirdparty.io.grpc.ServerCall;
@@ -39,11 +41,14 @@
next.startCall(call, headers)) {
@Override
public void onMessage(ReqT message) {
- try (Scope scope = TracingUtil
- .importAndCreateScope(
+ Span span = TracingUtil
+ .importAndCreateSpan(
call.getMethodDescriptor().getFullMethodName(),
- headers.get(GrpcClientInterceptor.TRACING_HEADER))) {
+ headers.get(GrpcClientInterceptor.TRACING_HEADER));
+ try (Scope scope = GlobalTracer.get().activateSpan(span)) {
super.onMessage(message);
+ } finally {
+ span.finish();
}
}
};
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TraceAllMethod.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TraceAllMethod.java
index 6dc9f96..2328094 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TraceAllMethod.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TraceAllMethod.java
@@ -25,6 +25,7 @@
import java.util.Map.Entry;
import io.opentracing.Scope;
+import io.opentracing.Span;
import io.opentracing.util.GlobalTracer;
/**
@@ -64,9 +65,10 @@
method.getName());
}
- try (Scope scope = GlobalTracer.get().buildSpan(
+ Span span = GlobalTracer.get().buildSpan(
name + "." + method.getName())
- .startActive(true)) {
+ .start();
+ try (Scope scope = GlobalTracer.get().activateSpan(span)) {
try {
return delegateMethod.invoke(delegate, args);
} catch (Exception ex) {
@@ -75,6 +77,8 @@
} else {
throw ex;
}
+ } finally {
+ span.finish();
}
}
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java
index fcfa613..fbab712 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java
@@ -17,7 +17,13 @@
*/
package org.apache.hadoop.hdds.tracing;
+import java.io.IOException;
import java.lang.reflect.Proxy;
+import java.util.function.Supplier;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.function.SupplierWithIOException;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import io.jaegertracing.Configuration;
import io.jaegertracing.internal.JaegerTracer;
@@ -27,8 +33,6 @@
import io.opentracing.Tracer;
import io.opentracing.util.GlobalTracer;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-
/**
* Utility class to collect all the tracing helper methods.
*/
@@ -43,7 +47,7 @@
* Initialize the tracing with the given service name.
*/
public static void initTracing(
- String serviceName, org.apache.hadoop.conf.Configuration conf) {
+ String serviceName, ConfigurationSource conf) {
if (!GlobalTracer.isRegistered() && isTracingEnabled(conf)) {
Configuration config = Configuration.fromEnv(serviceName);
JaegerTracer tracer = config.getTracerBuilder()
@@ -85,11 +89,11 @@
*
* @return OpenTracing scope.
*/
- public static Scope importAndCreateScope(String name, String encodedParent) {
+ public static Span importAndCreateSpan(String name, String encodedParent) {
Tracer tracer = GlobalTracer.get();
return tracer.buildSpan(name)
.asChildOf(extractParent(encodedParent, tracer))
- .startActive(true);
+ .start();
}
private static SpanContext extractParent(String parent, Tracer tracer) {
@@ -116,7 +120,7 @@
* calls to the delegate and also enables tracing.
*/
public static <T> T createProxy(
- T delegate, Class<T> itf, org.apache.hadoop.conf.Configuration conf) {
+ T delegate, Class<T> itf, ConfigurationSource conf) {
if (!isTracingEnabled(conf)) {
return delegate;
}
@@ -126,11 +130,78 @@
new TraceAllMethod<>(delegate, itf.getSimpleName())));
}
- private static boolean isTracingEnabled(
- org.apache.hadoop.conf.Configuration conf) {
+ public static boolean isTracingEnabled(
+ ConfigurationSource conf) {
return conf.getBoolean(
- ScmConfigKeys.HDDS_TRACING_ENABLED,
- ScmConfigKeys.HDDS_TRACING_ENABLED_DEFAULT);
+ ScmConfigKeys.HDDS_TRACING_ENABLED,
+ ScmConfigKeys.HDDS_TRACING_ENABLED_DEFAULT);
}
+ /**
+ * Execute a new function inside an activated span.
+ */
+ public static <R> R executeInNewSpan(String spanName,
+ SupplierWithIOException<R> supplier)
+ throws IOException {
+ Span span = GlobalTracer.get()
+ .buildSpan(spanName).start();
+ return executeInSpan(span, supplier);
+ }
+
+ /**
+ * Execute a new function inside an activated span.
+ */
+ public static <R> R executeInNewSpan(String spanName,
+ Supplier<R> supplier) {
+ Span span = GlobalTracer.get()
+ .buildSpan(spanName).start();
+ try (Scope scope = GlobalTracer.get().activateSpan(span)) {
+ return supplier.get();
+ } catch (Exception ex) {
+ span.setTag("failed", true);
+ throw ex;
+ } finally {
+ span.finish();
+ }
+ }
+
+ /**
+ * Execute a new function a given span.
+ */
+ private static <R> R executeInSpan(Span span,
+ SupplierWithIOException<R> supplier) throws IOException {
+ try (Scope scope = GlobalTracer.get().activateSpan(span)) {
+ return supplier.get();
+ } catch (Exception ex) {
+ span.setTag("failed", true);
+ throw ex;
+ } finally {
+ span.finish();
+ }
+ }
+
+ /**
+ * Execute a new function as a child span of the parent.
+ */
+ public static <R> R executeAsChildSpan(String spanName, String parentName,
+ SupplierWithIOException<R> supplier) throws IOException {
+ Span span = TracingUtil.importAndCreateSpan(spanName, parentName);
+ return executeInSpan(span, supplier);
+ }
+
+
+ /**
+ * Create an active span with auto-close at finish.
+ * <p>
+ * This is a simplified way to use span as there is no way to add any tag
+ * in case of Exceptions.
+ */
+ public static AutoCloseable createActivatedSpan(String spanName) {
+ Span span = GlobalTracer.get().buildSpan(spanName).start();
+ Scope scope = GlobalTracer.get().activateSpan(span);
+ return () -> {
+ scope.close();
+ span.finish();
+ };
+ }
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/LegacyHadoopConfigurationSource.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/LegacyHadoopConfigurationSource.java
new file mode 100644
index 0000000..44f2f9e
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/LegacyHadoopConfigurationSource.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.utils;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+
+/**
+ * Configuration source to wrap Hadoop Configuration object.
+ */
+public class LegacyHadoopConfigurationSource implements ConfigurationSource {
+
+ private Configuration configuration;
+
+ public LegacyHadoopConfigurationSource(
+ Configuration configuration) {
+ this.configuration = configuration;
+ }
+
+ @Override
+ public String get(String key) {
+ return configuration.getRaw(key);
+ }
+
+ @Override
+ public char[] getPassword(String key) throws IOException {
+ return configuration.getPassword(key);
+ }
+
+ @Override
+ public Collection<String> getConfigKeys() {
+ return configuration.getPropsWithPrefix("").keySet();
+ }
+
+ @Override
+ public void set(String key, String value) {
+ configuration.set(key, value);
+ }
+
+ /**
+ * Helper method to get original Hadoop configuration for legacy Hadoop
+ * libraries.
+ * <p>
+ * It can work on server side but not on client side where we might have
+ * different configuration.
+ */
+ public static Configuration asHadoopConfiguration(
+ ConfigurationSource config) {
+ if (config instanceof Configuration) {
+ return (Configuration) config;
+ } else if (config instanceof LegacyHadoopConfigurationSource) {
+ return ((LegacyHadoopConfigurationSource) config).configuration;
+ } else {
+ throw new IllegalArgumentException(
+ "Core Hadoop code requires real Hadoop configuration");
+ }
+ }
+
+ public Configuration getOriginalHadoopConfiguration() {
+ return configuration;
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index a83aa61..1ecaa0a 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -149,6 +149,17 @@
public static final TimeDuration OZONE_CLIENT_RETRY_INTERVAL_DEFAULT =
TimeDuration.valueOf(0, TimeUnit.MILLISECONDS);
+ /**
+ * If this value is true, when the client calls the flush() method,
+ * it checks whether the data in the buffer is greater than
+ * OZONE_CLIENT_STREAM_BUFFER_SIZE_DEFAULT. If greater than,
+ * send the data in the buffer to the datanode.
+ * */
+ public static final String OZONE_CLIENT_STREAM_BUFFER_FLUSH_DELAY =
+ "ozone.client.stream.buffer.flush.delay";
+ public static final boolean OOZONE_CLIENT_STREAM_BUFFER_FLUSH_DELAY_DEFAULT =
+ false;
+
// This defines the overall connection limit for the connection pool used in
// RestClient.
public static final String OZONE_REST_CLIENT_HTTP_CONNECTION_MAX =
@@ -347,6 +358,10 @@
"ozone.security.enabled";
public static final boolean OZONE_SECURITY_ENABLED_DEFAULT = false;
+ public static final String OZONE_HTTP_SECURITY_ENABLED_KEY =
+ "ozone.security.http.kerberos.enabled";
+ public static final boolean OZONE_HTTP_SECURITY_ENABLED_DEFAULT = false;
+
public static final String OZONE_CONTAINER_COPY_WORKDIR =
"hdds.datanode.replication.work.dir";
@@ -375,6 +390,10 @@
"ozone.acl.enabled";
public static final boolean OZONE_ACL_ENABLED_DEFAULT =
false;
+ public static final String OZONE_S3_VOLUME_NAME =
+ "ozone.s3g.volume.name";
+ public static final String OZONE_S3_VOLUME_NAME_DEFAULT =
+ "s3v";
public static final String OZONE_S3_AUTHINFO_MAX_LIFETIME_KEY =
"ozone.s3.token.max.lifetime";
public static final String OZONE_S3_AUTHINFO_MAX_LIFETIME_KEY_DEFAULT = "3m";
@@ -388,14 +407,10 @@
"ozone.client.failover.max.attempts";
public static final int OZONE_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT =
15;
- public static final String OZONE_CLIENT_FAILOVER_SLEEP_BASE_MILLIS_KEY =
- "ozone.client.failover.sleep.base.millis";
- public static final int OZONE_CLIENT_FAILOVER_SLEEP_BASE_MILLIS_DEFAULT =
- 500;
- public static final String OZONE_CLIENT_FAILOVER_SLEEP_MAX_MILLIS_KEY =
- "ozone.client.failover.sleep.max.millis";
- public static final int OZONE_CLIENT_FAILOVER_SLEEP_MAX_MILLIS_DEFAULT =
- 15000;
+ public static final String OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_KEY =
+ "ozone.client.wait.between.retries.millis";
+ public static final long OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_DEFAULT =
+ 2000;
public static final String OZONE_FREON_HTTP_ENABLED_KEY =
"ozone.freon.http.enabled";
@@ -413,10 +428,15 @@
public static final int OZONE_FREON_HTTPS_BIND_PORT_DEFAULT = 9885;
public static final String
OZONE_FREON_HTTP_KERBEROS_PRINCIPAL_KEY =
- "ozone.freon.http.kerberos.principal";
+ "ozone.freon.http.auth.kerberos.principal";
public static final String
OZONE_FREON_HTTP_KERBEROS_KEYTAB_FILE_KEY =
- "ozone.freon.http.kerberos.keytab";
+ "ozone.freon.http.auth.kerberos.keytab";
+ public static final String OZONE_FREON_HTTP_AUTH_TYPE =
+ "ozone.freon.http.auth.type";
+ public static final String OZONE_FREON_HTTP_AUTH_CONFIG_PREFIX =
+ "ozone.freon.http.auth.";
+
public static final String OZONE_NETWORK_TOPOLOGY_AWARE_READ_KEY =
"ozone.network.topology.aware.read";
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index c6589d4..8a57e65 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.ozone;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.ratis.thirdparty.io.grpc.Context;
import org.apache.ratis.thirdparty.io.grpc.Metadata;
@@ -88,6 +89,8 @@
public static final String OZONE_RPC_SCHEME = "o3";
public static final String OZONE_HTTP_SCHEME = "http";
public static final String OZONE_URI_DELIMITER = "/";
+ public static final String OZONE_ROOT = OZONE_URI_DELIMITER;
+
public static final String CONTAINER_EXTENSION = ".container";
public static final String CONTAINER_META = ".meta";
@@ -114,14 +117,9 @@
*/
public static final String CONTAINER_DB_SUFFIX = "container.db";
public static final String PIPELINE_DB_SUFFIX = "pipeline.db";
- public static final String SCM_CONTAINER_DB = "scm-" + CONTAINER_DB_SUFFIX;
- public static final String SCM_PIPELINE_DB = "scm-" + PIPELINE_DB_SUFFIX;
public static final String DN_CONTAINER_DB = "-dn-"+ CONTAINER_DB_SUFFIX;
- public static final String DELETED_BLOCK_DB = "deletedBlock.db";
public static final String OM_DB_NAME = "om.db";
public static final String OM_DB_BACKUP_PREFIX = "om.db.backup.";
- public static final String OZONE_MANAGER_TOKEN_DB_NAME = "om-token.db";
- public static final String SCM_DB_NAME = "scm.db";
public static final String STORAGE_DIR_CHUNKS = "chunks";
public static final String OZONE_DB_CHECKPOINT_REQUEST_FLUSH =
@@ -143,6 +141,25 @@
public static final String DELETE_TRANSACTION_KEY_PREFIX = "#delTX#";
public static final String BLOCK_COMMIT_SEQUENCE_ID_PREFIX = "#BCSID";
+ public static final String BLOCK_COUNT = "#BLOCKCOUNT";
+ public static final String CONTAINER_BYTES_USED = "#BYTESUSED";
+ public static final String PENDING_DELETE_BLOCK_COUNT =
+ "#PENDINGDELETEBLOCKCOUNT";
+
+
+ public static final byte[] DB_BLOCK_COUNT_KEY =
+ DFSUtil.string2Bytes(OzoneConsts.BLOCK_COUNT);
+ public static final byte[] DB_CONTAINER_BYTES_USED_KEY =
+ DFSUtil.string2Bytes(OzoneConsts.CONTAINER_BYTES_USED);
+ public static final byte[] DB_PENDING_DELETE_BLOCK_COUNT_KEY =
+ DFSUtil.string2Bytes(PENDING_DELETE_BLOCK_COUNT);
+ public static final byte[] DB_CONTAINER_DELETE_TRANSACTION_KEY =
+ DFSUtil.string2Bytes(DELETE_TRANSACTION_KEY_PREFIX);
+ public static final byte[] DB_BLOCK_COMMIT_SEQUENCE_ID_KEY =
+ DFSUtil.string2Bytes(BLOCK_COMMIT_SEQUENCE_ID_PREFIX);
+
+
+
/**
* OM LevelDB prefixes.
*
@@ -239,7 +256,6 @@
public static final String KEY = "key";
public static final String SRC_KEY = "srcKey";
public static final String DST_KEY = "dstKey";
- public static final String QUOTA = "quota";
public static final String QUOTA_IN_BYTES = "quotaInBytes";
public static final String OBJECT_ID = "objectID";
public static final String UPDATE_ID = "updateID";
@@ -275,6 +291,7 @@
public static final String S3_GETSECRET_USER = "S3GetSecretUser";
public static final String MULTIPART_UPLOAD_PART_NUMBER = "partNumber";
public static final String MULTIPART_UPLOAD_PART_NAME = "partName";
+ public static final String BUCKET_ENCRYPTION_KEY = "bucketEncryptionKey";
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java
index 77b7062..726862c 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java
@@ -18,16 +18,6 @@
package org.apache.hadoop.ozone;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
-
-import org.apache.commons.validator.routines.InetAddressValidator;
-import org.apache.hadoop.hdds.annotation.InterfaceAudience;
-import org.apache.hadoop.hdds.annotation.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.File;
import java.io.IOException;
import java.net.InetAddress;
@@ -40,6 +30,18 @@
import java.util.List;
import java.util.Set;
+import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.annotation.InterfaceStability;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+
+import org.apache.commons.validator.routines.InetAddressValidator;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_HTTP_SECURITY_ENABLED_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_HTTP_SECURITY_ENABLED_KEY;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* Ozone security Util class.
*/
@@ -56,11 +58,17 @@
private OzoneSecurityUtil() {
}
- public static boolean isSecurityEnabled(Configuration conf) {
+ public static boolean isSecurityEnabled(ConfigurationSource conf) {
return conf.getBoolean(OZONE_SECURITY_ENABLED_KEY,
OZONE_SECURITY_ENABLED_DEFAULT);
}
+ public static boolean isHttpSecurityEnabled(ConfigurationSource conf) {
+ return isSecurityEnabled(conf) &&
+ conf.getBoolean(OZONE_HTTP_SECURITY_ENABLED_KEY,
+ OZONE_HTTP_SECURITY_ENABLED_DEFAULT);
+ }
+
/**
* Returns Keys status.
*
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java
index fada2d8..102d47a 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java
@@ -43,7 +43,8 @@
SORT_DATANODE,
START_REPLICATION_MANAGER,
STOP_REPLICATION_MANAGER,
- GET_REPLICATION_MANAGER_STATUS;
+ GET_REPLICATION_MANAGER_STATUS,
+ GET_CONTAINER_WITH_PIPELINE_BATCH;
@Override
public String getAction() {
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/conf/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/conf/package-info.java
deleted file mode 100644
index f4e6a38..0000000
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/conf/package-info.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- This package contains class related to configuration.
- */
-
-package org.apache.hadoop.ozone.conf;
-
-
-
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/LockManager.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/LockManager.java
index 84aca6d..ac6e9a1 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/LockManager.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/lock/LockManager.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.ozone.lock;
import org.apache.commons.pool2.impl.GenericObjectPool;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -44,7 +44,7 @@
*
* @param conf Configuration object
*/
- public LockManager(final Configuration conf) {
+ public LockManager(final ConfigurationSource conf) {
this(conf, false);
}
@@ -55,7 +55,7 @@
* @param conf Configuration object
* @param fair - true to use fair lock ordering, else non-fair lock ordering.
*/
- public LockManager(final Configuration conf, boolean fair) {
+ public LockManager(final ConfigurationSource conf, boolean fair) {
lockPool =
new GenericObjectPool<>(new PooledLockFactory(fair));
lockPool.setMaxTotal(-1);
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/package-info.java
index db399db..2e4d8b8 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/package-info.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/package-info.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -15,21 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.ozone;
/**
- This package contains class that support ozone implementation on the datanode
- side.
-
- Main parts of ozone on datanode are:
-
- 1. REST Interface - This code lives under the web directory and listens to the
- WebHDFS port.
-
- 2. Datanode container classes: This support persistence of ozone objects on
- datanode. These classes live under container directory.
-
- 3. Client and Shell: We also support a ozone REST client lib, they are under
- web/client and web/ozShell.
-
+ * This package contains class that support ozone implementation on the datanode
+ * side. Datanode container classes support persistence of ozone objects on
+ * datanode. These classes live under container directory.
*/
+package org.apache.hadoop.ozone;
diff --git a/hadoop-hdds/common/src/main/proto/FSProtos.proto b/hadoop-hdds/common/src/main/proto/FSProtos.proto
deleted file mode 100644
index c3b768a..0000000
--- a/hadoop-hdds/common/src/main/proto/FSProtos.proto
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
- * for what changes are allowed for a *stable* .proto interface.
- */
-
-option java_package = "org.apache.hadoop.fs";
-option java_outer_classname = "FSProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-package hadoop.fs;
-
-message FsPermissionProto {
- required uint32 perm = 1; // UNIX-style mode bits
-}
-
-/*
- * FileStatus encoding. Field IDs match those from HdfsFileStatusProto, but
- * cross-serialization is not an explicitly supported use case. Unlike HDFS,
- * most fields are optional and do not define defaults.
- */
-message FileStatusProto {
- enum FileType {
- FT_DIR = 1;
- FT_FILE = 2;
- FT_SYMLINK = 3;
- }
- enum Flags {
- HAS_ACL = 0x01; // has ACLs
- HAS_CRYPT = 0x02; // encrypted
- HAS_EC = 0x04; // erasure coded
- SNAPSHOT_ENABLED = 0x08; // snapshot enabled
- }
- required FileType fileType = 1;
- required string path = 2;
- optional uint64 length = 3;
- optional FsPermissionProto permission = 4;
- optional string owner = 5;
- optional string group = 6;
- optional uint64 modification_time = 7;
- optional uint64 access_time = 8;
- optional string symlink = 9;
- optional uint32 block_replication = 10;
- optional uint64 block_size = 11;
- // locations = 12
- // alias = 13
- // childrenNum = 14
- optional bytes encryption_data = 15;
- // storagePolicy = 16
- optional bytes ec_data = 17;
- optional uint32 flags = 18 [default = 0];
-}
-
-/**
- * Placeholder type for consistent basic FileSystem operations.
- */
-message LocalFileSystemPathHandleProto {
- optional uint64 mtime = 1;
- optional string path = 2;
-}
diff --git a/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto b/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto
index 88df770..2bb9990 100644
--- a/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto
+++ b/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto
@@ -60,6 +60,7 @@
optional StopReplicationManagerRequestProto stopReplicationManagerRequest = 22;
optional ReplicationManagerStatusRequestProto seplicationManagerStatusRequest = 23;
optional GetPipelineRequestProto getPipelineRequest = 24;
+ optional GetContainerWithPipelineBatchRequestProto getContainerWithPipelineBatchRequest = 25;
}
@@ -93,6 +94,7 @@
optional StopReplicationManagerResponseProto stopReplicationManagerResponse = 22;
optional ReplicationManagerStatusResponseProto replicationManagerStatusResponse = 23;
optional GetPipelineResponseProto getPipelineResponse = 24;
+ optional GetContainerWithPipelineBatchResponseProto getContainerWithPipelineBatchResponse = 25;
enum Status {
OK = 1;
CONTAINER_ALREADY_EXISTS = 2;
@@ -121,6 +123,7 @@
StopReplicationManager = 17;
GetReplicationManagerStatus = 18;
GetPipeline = 19;
+ GetContainerWithPipelineBatch = 20;
}
/**
@@ -168,6 +171,15 @@
required ContainerWithPipeline containerWithPipeline = 1;
}
+message GetContainerWithPipelineBatchRequestProto {
+ repeated int64 containerIDs = 1;
+ optional string traceID = 2;
+}
+
+message GetContainerWithPipelineBatchResponseProto {
+ repeated ContainerWithPipeline containerWithPipelines = 1;
+}
+
message SCMListContainerRequestProto {
required uint32 count = 1;
optional uint64 startContainerID = 2;
diff --git a/hadoop-hdds/common/src/main/proto/proto.lock b/hadoop-hdds/common/src/main/proto/proto.lock
new file mode 100644
index 0000000..02749d7
--- /dev/null
+++ b/hadoop-hdds/common/src/main/proto/proto.lock
@@ -0,0 +1,3640 @@
+{
+ "definitions": [
+ {
+ "protopath": "DatanodeContainerProtocol.proto",
+ "def": {
+ "enums": [
+ {
+ "name": "Type",
+ "enum_fields": [
+ {
+ "name": "CreateContainer",
+ "integer": 1
+ },
+ {
+ "name": "ReadContainer",
+ "integer": 2
+ },
+ {
+ "name": "UpdateContainer",
+ "integer": 3
+ },
+ {
+ "name": "DeleteContainer",
+ "integer": 4
+ },
+ {
+ "name": "ListContainer",
+ "integer": 5
+ },
+ {
+ "name": "PutBlock",
+ "integer": 6
+ },
+ {
+ "name": "GetBlock",
+ "integer": 7
+ },
+ {
+ "name": "DeleteBlock",
+ "integer": 8
+ },
+ {
+ "name": "ListBlock",
+ "integer": 9
+ },
+ {
+ "name": "ReadChunk",
+ "integer": 10
+ },
+ {
+ "name": "DeleteChunk",
+ "integer": 11
+ },
+ {
+ "name": "WriteChunk",
+ "integer": 12
+ },
+ {
+ "name": "ListChunk",
+ "integer": 13
+ },
+ {
+ "name": "CompactChunk",
+ "integer": 14
+ },
+ {
+ "name": "PutSmallFile",
+ "integer": 15
+ },
+ {
+ "name": "GetSmallFile",
+ "integer": 16
+ },
+ {
+ "name": "CloseContainer",
+ "integer": 17
+ },
+ {
+ "name": "GetCommittedBlockLength",
+ "integer": 18
+ }
+ ]
+ },
+ {
+ "name": "Result",
+ "enum_fields": [
+ {
+ "name": "SUCCESS",
+ "integer": 1
+ },
+ {
+ "name": "UNSUPPORTED_REQUEST",
+ "integer": 2
+ },
+ {
+ "name": "MALFORMED_REQUEST",
+ "integer": 3
+ },
+ {
+ "name": "CONTAINER_INTERNAL_ERROR",
+ "integer": 4
+ },
+ {
+ "name": "INVALID_CONFIG",
+ "integer": 5
+ },
+ {
+ "name": "INVALID_FILE_HASH_FOUND",
+ "integer": 6
+ },
+ {
+ "name": "CONTAINER_EXISTS",
+ "integer": 7
+ },
+ {
+ "name": "NO_SUCH_ALGORITHM",
+ "integer": 8
+ },
+ {
+ "name": "CONTAINER_NOT_FOUND",
+ "integer": 9
+ },
+ {
+ "name": "IO_EXCEPTION",
+ "integer": 10
+ },
+ {
+ "name": "UNABLE_TO_READ_METADATA_DB",
+ "integer": 11
+ },
+ {
+ "name": "NO_SUCH_BLOCK",
+ "integer": 12
+ },
+ {
+ "name": "OVERWRITE_FLAG_REQUIRED",
+ "integer": 13
+ },
+ {
+ "name": "UNABLE_TO_FIND_DATA_DIR",
+ "integer": 14
+ },
+ {
+ "name": "INVALID_WRITE_SIZE",
+ "integer": 15
+ },
+ {
+ "name": "CHECKSUM_MISMATCH",
+ "integer": 16
+ },
+ {
+ "name": "UNABLE_TO_FIND_CHUNK",
+ "integer": 17
+ },
+ {
+ "name": "PROTOC_DECODING_ERROR",
+ "integer": 18
+ },
+ {
+ "name": "INVALID_ARGUMENT",
+ "integer": 19
+ },
+ {
+ "name": "PUT_SMALL_FILE_ERROR",
+ "integer": 20
+ },
+ {
+ "name": "GET_SMALL_FILE_ERROR",
+ "integer": 21
+ },
+ {
+ "name": "CLOSED_CONTAINER_IO",
+ "integer": 22
+ },
+ {
+ "name": "ERROR_IN_COMPACT_DB",
+ "integer": 24
+ },
+ {
+ "name": "UNCLOSED_CONTAINER_IO",
+ "integer": 25
+ },
+ {
+ "name": "DELETE_ON_OPEN_CONTAINER",
+ "integer": 26
+ },
+ {
+ "name": "CLOSED_CONTAINER_RETRY",
+ "integer": 27
+ },
+ {
+ "name": "INVALID_CONTAINER_STATE",
+ "integer": 28
+ },
+ {
+ "name": "DISK_OUT_OF_SPACE",
+ "integer": 29
+ },
+ {
+ "name": "CONTAINER_ALREADY_EXISTS",
+ "integer": 30
+ },
+ {
+ "name": "CONTAINER_METADATA_ERROR",
+ "integer": 31
+ },
+ {
+ "name": "CONTAINER_FILES_CREATE_ERROR",
+ "integer": 32
+ },
+ {
+ "name": "CONTAINER_CHECKSUM_ERROR",
+ "integer": 33
+ },
+ {
+ "name": "UNKNOWN_CONTAINER_TYPE",
+ "integer": 34
+ },
+ {
+ "name": "BLOCK_NOT_COMMITTED",
+ "integer": 35
+ },
+ {
+ "name": "CONTAINER_UNHEALTHY",
+ "integer": 36
+ },
+ {
+ "name": "UNKNOWN_BCSID",
+ "integer": 37
+ },
+ {
+ "name": "BCSID_MISMATCH",
+ "integer": 38
+ },
+ {
+ "name": "CONTAINER_NOT_OPEN",
+ "integer": 39
+ },
+ {
+ "name": "CONTAINER_MISSING",
+ "integer": 40
+ },
+ {
+ "name": "BLOCK_TOKEN_VERIFICATION_FAILED",
+ "integer": 41
+ },
+ {
+ "name": "ERROR_IN_DB_SYNC",
+ "integer": 42
+ }
+ ]
+ },
+ {
+ "name": "ContainerDataProto.State",
+ "enum_fields": [
+ {
+ "name": "OPEN",
+ "integer": 1
+ },
+ {
+ "name": "CLOSING",
+ "integer": 2
+ },
+ {
+ "name": "QUASI_CLOSED",
+ "integer": 3
+ },
+ {
+ "name": "CLOSED",
+ "integer": 4
+ },
+ {
+ "name": "UNHEALTHY",
+ "integer": 5
+ },
+ {
+ "name": "INVALID",
+ "integer": 6
+ }
+ ]
+ },
+ {
+ "name": "ContainerType",
+ "enum_fields": [
+ {
+ "name": "KeyValueContainer",
+ "integer": 1
+ }
+ ]
+ },
+ {
+ "name": "ChecksumType",
+ "enum_fields": [
+ {
+ "name": "NONE",
+ "integer": 1
+ },
+ {
+ "name": "CRC32",
+ "integer": 2
+ },
+ {
+ "name": "CRC32C",
+ "integer": 3
+ },
+ {
+ "name": "SHA256",
+ "integer": 4
+ },
+ {
+ "name": "MD5",
+ "integer": 5
+ }
+ ]
+ }
+ ],
+ "messages": [
+ {
+ "name": "DatanodeBlockID",
+ "fields": [
+ {
+ "id": 1,
+ "name": "containerID",
+ "type": "int64"
+ },
+ {
+ "id": 2,
+ "name": "localID",
+ "type": "int64"
+ },
+ {
+ "id": 3,
+ "name": "blockCommitSequenceId",
+ "type": "uint64",
+ "options": [
+ {
+ "name": "default",
+ "value": "0"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "KeyValue",
+ "fields": [
+ {
+ "id": 1,
+ "name": "key",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "value",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "ContainerCommandRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "cmdType",
+ "type": "Type"
+ },
+ {
+ "id": 2,
+ "name": "traceID",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "containerID",
+ "type": "int64"
+ },
+ {
+ "id": 4,
+ "name": "datanodeUuid",
+ "type": "string"
+ },
+ {
+ "id": 5,
+ "name": "pipelineID",
+ "type": "string"
+ },
+ {
+ "id": 6,
+ "name": "createContainer",
+ "type": "CreateContainerRequestProto"
+ },
+ {
+ "id": 7,
+ "name": "readContainer",
+ "type": "ReadContainerRequestProto"
+ },
+ {
+ "id": 8,
+ "name": "updateContainer",
+ "type": "UpdateContainerRequestProto"
+ },
+ {
+ "id": 9,
+ "name": "deleteContainer",
+ "type": "DeleteContainerRequestProto"
+ },
+ {
+ "id": 10,
+ "name": "listContainer",
+ "type": "ListContainerRequestProto"
+ },
+ {
+ "id": 11,
+ "name": "closeContainer",
+ "type": "CloseContainerRequestProto"
+ },
+ {
+ "id": 12,
+ "name": "putBlock",
+ "type": "PutBlockRequestProto"
+ },
+ {
+ "id": 13,
+ "name": "getBlock",
+ "type": "GetBlockRequestProto"
+ },
+ {
+ "id": 14,
+ "name": "deleteBlock",
+ "type": "DeleteBlockRequestProto"
+ },
+ {
+ "id": 15,
+ "name": "listBlock",
+ "type": "ListBlockRequestProto"
+ },
+ {
+ "id": 16,
+ "name": "readChunk",
+ "type": "ReadChunkRequestProto"
+ },
+ {
+ "id": 17,
+ "name": "writeChunk",
+ "type": "WriteChunkRequestProto"
+ },
+ {
+ "id": 18,
+ "name": "deleteChunk",
+ "type": "DeleteChunkRequestProto"
+ },
+ {
+ "id": 19,
+ "name": "listChunk",
+ "type": "ListChunkRequestProto"
+ },
+ {
+ "id": 20,
+ "name": "putSmallFile",
+ "type": "PutSmallFileRequestProto"
+ },
+ {
+ "id": 21,
+ "name": "getSmallFile",
+ "type": "GetSmallFileRequestProto"
+ },
+ {
+ "id": 22,
+ "name": "getCommittedBlockLength",
+ "type": "GetCommittedBlockLengthRequestProto"
+ },
+ {
+ "id": 23,
+ "name": "encodedToken",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "ContainerCommandResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "cmdType",
+ "type": "Type"
+ },
+ {
+ "id": 2,
+ "name": "traceID",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "result",
+ "type": "Result"
+ },
+ {
+ "id": 4,
+ "name": "message",
+ "type": "string"
+ },
+ {
+ "id": 5,
+ "name": "createContainer",
+ "type": "CreateContainerResponseProto"
+ },
+ {
+ "id": 6,
+ "name": "readContainer",
+ "type": "ReadContainerResponseProto"
+ },
+ {
+ "id": 7,
+ "name": "updateContainer",
+ "type": "UpdateContainerResponseProto"
+ },
+ {
+ "id": 8,
+ "name": "deleteContainer",
+ "type": "DeleteContainerResponseProto"
+ },
+ {
+ "id": 9,
+ "name": "listContainer",
+ "type": "ListContainerResponseProto"
+ },
+ {
+ "id": 10,
+ "name": "closeContainer",
+ "type": "CloseContainerResponseProto"
+ },
+ {
+ "id": 11,
+ "name": "putBlock",
+ "type": "PutBlockResponseProto"
+ },
+ {
+ "id": 12,
+ "name": "getBlock",
+ "type": "GetBlockResponseProto"
+ },
+ {
+ "id": 13,
+ "name": "deleteBlock",
+ "type": "DeleteBlockResponseProto"
+ },
+ {
+ "id": 14,
+ "name": "listBlock",
+ "type": "ListBlockResponseProto"
+ },
+ {
+ "id": 15,
+ "name": "writeChunk",
+ "type": "WriteChunkResponseProto"
+ },
+ {
+ "id": 16,
+ "name": "readChunk",
+ "type": "ReadChunkResponseProto"
+ },
+ {
+ "id": 17,
+ "name": "deleteChunk",
+ "type": "DeleteChunkResponseProto"
+ },
+ {
+ "id": 18,
+ "name": "listChunk",
+ "type": "ListChunkResponseProto"
+ },
+ {
+ "id": 19,
+ "name": "putSmallFile",
+ "type": "PutSmallFileResponseProto"
+ },
+ {
+ "id": 20,
+ "name": "getSmallFile",
+ "type": "GetSmallFileResponseProto"
+ },
+ {
+ "id": 21,
+ "name": "getCommittedBlockLength",
+ "type": "GetCommittedBlockLengthResponseProto"
+ }
+ ]
+ },
+ {
+ "name": "ContainerDataProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "containerID",
+ "type": "int64"
+ },
+ {
+ "id": 2,
+ "name": "metadata",
+ "type": "KeyValue",
+ "is_repeated": true
+ },
+ {
+ "id": 4,
+ "name": "containerPath",
+ "type": "string"
+ },
+ {
+ "id": 6,
+ "name": "bytesUsed",
+ "type": "int64"
+ },
+ {
+ "id": 7,
+ "name": "size",
+ "type": "int64"
+ },
+ {
+ "id": 8,
+ "name": "blockCount",
+ "type": "int64"
+ },
+ {
+ "id": 9,
+ "name": "state",
+ "type": "State",
+ "options": [
+ {
+ "name": "default",
+ "value": "OPEN"
+ }
+ ]
+ },
+ {
+ "id": 10,
+ "name": "containerType",
+ "type": "ContainerType",
+ "options": [
+ {
+ "name": "default",
+ "value": "KeyValueContainer"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "Container2BCSIDMapProto",
+ "maps": [
+ {
+ "key_type": "int64",
+ "field": {
+ "id": 1,
+ "name": "container2BCSID",
+ "type": "int64"
+ }
+ }
+ ]
+ },
+ {
+ "name": "CreateContainerRequestProto",
+ "fields": [
+ {
+ "id": 2,
+ "name": "metadata",
+ "type": "KeyValue",
+ "is_repeated": true
+ },
+ {
+ "id": 3,
+ "name": "containerType",
+ "type": "ContainerType",
+ "options": [
+ {
+ "name": "default",
+ "value": "KeyValueContainer"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "CreateContainerResponseProto"
+ },
+ {
+ "name": "ReadContainerRequestProto"
+ },
+ {
+ "name": "ReadContainerResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "containerData",
+ "type": "ContainerDataProto"
+ }
+ ]
+ },
+ {
+ "name": "UpdateContainerRequestProto",
+ "fields": [
+ {
+ "id": 2,
+ "name": "metadata",
+ "type": "KeyValue",
+ "is_repeated": true
+ },
+ {
+ "id": 3,
+ "name": "forceUpdate",
+ "type": "bool",
+ "options": [
+ {
+ "name": "default",
+ "value": "false"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "UpdateContainerResponseProto"
+ },
+ {
+ "name": "DeleteContainerRequestProto",
+ "fields": [
+ {
+ "id": 2,
+ "name": "forceDelete",
+ "type": "bool",
+ "options": [
+ {
+ "name": "default",
+ "value": "false"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "DeleteContainerResponseProto"
+ },
+ {
+ "name": "ListContainerRequestProto",
+ "fields": [
+ {
+ "id": 2,
+ "name": "count",
+ "type": "uint32"
+ }
+ ]
+ },
+ {
+ "name": "ListContainerResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "containerData",
+ "type": "ContainerDataProto",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "CloseContainerRequestProto"
+ },
+ {
+ "name": "CloseContainerResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "hash",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "containerID",
+ "type": "int64"
+ }
+ ]
+ },
+ {
+ "name": "BlockData",
+ "fields": [
+ {
+ "id": 1,
+ "name": "blockID",
+ "type": "DatanodeBlockID"
+ },
+ {
+ "id": 2,
+ "name": "flags",
+ "type": "int64"
+ },
+ {
+ "id": 3,
+ "name": "metadata",
+ "type": "KeyValue",
+ "is_repeated": true
+ },
+ {
+ "id": 4,
+ "name": "chunks",
+ "type": "ChunkInfo",
+ "is_repeated": true
+ },
+ {
+ "id": 5,
+ "name": "size",
+ "type": "int64"
+ }
+ ]
+ },
+ {
+ "name": "PutBlockRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "blockData",
+ "type": "BlockData"
+ },
+ {
+ "id": 2,
+ "name": "eof",
+ "type": "bool"
+ }
+ ]
+ },
+ {
+ "name": "PutBlockResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "committedBlockLength",
+ "type": "GetCommittedBlockLengthResponseProto"
+ }
+ ]
+ },
+ {
+ "name": "GetBlockRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "blockID",
+ "type": "DatanodeBlockID"
+ }
+ ]
+ },
+ {
+ "name": "GetBlockResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "blockData",
+ "type": "BlockData"
+ }
+ ]
+ },
+ {
+ "name": "DeleteBlockRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "blockID",
+ "type": "DatanodeBlockID"
+ }
+ ]
+ },
+ {
+ "name": "GetCommittedBlockLengthRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "blockID",
+ "type": "DatanodeBlockID"
+ }
+ ]
+ },
+ {
+ "name": "GetCommittedBlockLengthResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "blockID",
+ "type": "DatanodeBlockID"
+ },
+ {
+ "id": 2,
+ "name": "blockLength",
+ "type": "int64"
+ }
+ ]
+ },
+ {
+ "name": "DeleteBlockResponseProto"
+ },
+ {
+ "name": "ListBlockRequestProto",
+ "fields": [
+ {
+ "id": 2,
+ "name": "startLocalID",
+ "type": "int64"
+ },
+ {
+ "id": 3,
+ "name": "count",
+ "type": "uint32"
+ }
+ ]
+ },
+ {
+ "name": "ListBlockResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "blockData",
+ "type": "BlockData",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "ChunkInfo",
+ "fields": [
+ {
+ "id": 1,
+ "name": "chunkName",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "offset",
+ "type": "uint64"
+ },
+ {
+ "id": 3,
+ "name": "len",
+ "type": "uint64"
+ },
+ {
+ "id": 4,
+ "name": "metadata",
+ "type": "KeyValue",
+ "is_repeated": true
+ },
+ {
+ "id": 5,
+ "name": "checksumData",
+ "type": "ChecksumData"
+ }
+ ]
+ },
+ {
+ "name": "ChecksumData",
+ "fields": [
+ {
+ "id": 1,
+ "name": "type",
+ "type": "ChecksumType"
+ },
+ {
+ "id": 2,
+ "name": "bytesPerChecksum",
+ "type": "uint32"
+ },
+ {
+ "id": 3,
+ "name": "checksums",
+ "type": "bytes",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "WriteChunkRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "blockID",
+ "type": "DatanodeBlockID"
+ },
+ {
+ "id": 2,
+ "name": "chunkData",
+ "type": "ChunkInfo"
+ },
+ {
+ "id": 3,
+ "name": "data",
+ "type": "bytes"
+ }
+ ]
+ },
+ {
+ "name": "WriteChunkResponseProto"
+ },
+ {
+ "name": "ReadChunkRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "blockID",
+ "type": "DatanodeBlockID"
+ },
+ {
+ "id": 2,
+ "name": "chunkData",
+ "type": "ChunkInfo"
+ }
+ ]
+ },
+ {
+ "name": "ReadChunkResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "blockID",
+ "type": "DatanodeBlockID"
+ },
+ {
+ "id": 2,
+ "name": "chunkData",
+ "type": "ChunkInfo"
+ },
+ {
+ "id": 3,
+ "name": "data",
+ "type": "bytes"
+ }
+ ]
+ },
+ {
+ "name": "DeleteChunkRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "blockID",
+ "type": "DatanodeBlockID"
+ },
+ {
+ "id": 2,
+ "name": "chunkData",
+ "type": "ChunkInfo"
+ }
+ ]
+ },
+ {
+ "name": "DeleteChunkResponseProto"
+ },
+ {
+ "name": "ListChunkRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "blockID",
+ "type": "DatanodeBlockID"
+ },
+ {
+ "id": 2,
+ "name": "prevChunkName",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "count",
+ "type": "uint32"
+ }
+ ]
+ },
+ {
+ "name": "ListChunkResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "chunkData",
+ "type": "ChunkInfo",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "PutSmallFileRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "block",
+ "type": "PutBlockRequestProto"
+ },
+ {
+ "id": 2,
+ "name": "chunkInfo",
+ "type": "ChunkInfo"
+ },
+ {
+ "id": 3,
+ "name": "data",
+ "type": "bytes"
+ }
+ ]
+ },
+ {
+ "name": "PutSmallFileResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "committedBlockLength",
+ "type": "GetCommittedBlockLengthResponseProto"
+ }
+ ]
+ },
+ {
+ "name": "GetSmallFileRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "block",
+ "type": "GetBlockRequestProto"
+ }
+ ]
+ },
+ {
+ "name": "GetSmallFileResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "data",
+ "type": "ReadChunkResponseProto"
+ }
+ ]
+ },
+ {
+ "name": "CopyContainerRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "containerID",
+ "type": "int64"
+ },
+ {
+ "id": 2,
+ "name": "readOffset",
+ "type": "uint64"
+ },
+ {
+ "id": 3,
+ "name": "len",
+ "type": "uint64"
+ }
+ ]
+ },
+ {
+ "name": "CopyContainerResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "containerID",
+ "type": "int64"
+ },
+ {
+ "id": 2,
+ "name": "readOffset",
+ "type": "uint64"
+ },
+ {
+ "id": 3,
+ "name": "len",
+ "type": "uint64"
+ },
+ {
+ "id": 4,
+ "name": "eof",
+ "type": "bool"
+ },
+ {
+ "id": 5,
+ "name": "data",
+ "type": "bytes"
+ },
+ {
+ "id": 6,
+ "name": "checksum",
+ "type": "int64"
+ }
+ ]
+ }
+ ],
+ "services": [
+ {
+ "name": "XceiverClientProtocolService",
+ "rpcs": [
+ {
+ "name": "send",
+ "in_type": "ContainerCommandRequestProto",
+ "out_type": "ContainerCommandResponseProto",
+ "in_streamed": true,
+ "out_streamed": true
+ }
+ ]
+ },
+ {
+ "name": "IntraDatanodeProtocolService",
+ "rpcs": [
+ {
+ "name": "download",
+ "in_type": "CopyContainerRequestProto",
+ "out_type": "CopyContainerResponseProto",
+ "out_streamed": true
+ }
+ ]
+ }
+ ],
+ "package": {
+ "name": "hadoop.hdds.datanode"
+ },
+ "options": [
+ {
+ "name": "java_package",
+ "value": "org.apache.hadoop.hdds.protocol.datanode.proto"
+ },
+ {
+ "name": "java_outer_classname",
+ "value": "ContainerProtos"
+ },
+ {
+ "name": "java_generate_equals_and_hash",
+ "value": "true"
+ }
+ ]
+ }
+ },
+ {
+ "protopath": "SCMSecurityProtocol.proto",
+ "def": {
+ "enums": [
+ {
+ "name": "Type",
+ "enum_fields": [
+ {
+ "name": "GetDataNodeCertificate",
+ "integer": 1
+ },
+ {
+ "name": "GetOMCertificate",
+ "integer": 2
+ },
+ {
+ "name": "GetCertificate",
+ "integer": 3
+ },
+ {
+ "name": "GetCACertificate",
+ "integer": 4
+ }
+ ]
+ },
+ {
+ "name": "Status",
+ "enum_fields": [
+ {
+ "name": "OK",
+ "integer": 1
+ }
+ ]
+ },
+ {
+ "name": "SCMGetCertResponseProto.ResponseCode",
+ "enum_fields": [
+ {
+ "name": "success",
+ "integer": 1
+ },
+ {
+ "name": "authenticationFailed",
+ "integer": 2
+ },
+ {
+ "name": "invalidCSR",
+ "integer": 3
+ }
+ ]
+ }
+ ],
+ "messages": [
+ {
+ "name": "SCMSecurityRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "cmdType",
+ "type": "Type"
+ },
+ {
+ "id": 2,
+ "name": "traceID",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "getDataNodeCertRequest",
+ "type": "SCMGetDataNodeCertRequestProto"
+ },
+ {
+ "id": 4,
+ "name": "getOMCertRequest",
+ "type": "SCMGetOMCertRequestProto"
+ },
+ {
+ "id": 5,
+ "name": "getCertificateRequest",
+ "type": "SCMGetCertificateRequestProto"
+ },
+ {
+ "id": 6,
+ "name": "getCACertificateRequest",
+ "type": "SCMGetCACertificateRequestProto"
+ }
+ ]
+ },
+ {
+ "name": "SCMSecurityResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "cmdType",
+ "type": "Type"
+ },
+ {
+ "id": 2,
+ "name": "traceID",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "success",
+ "type": "bool",
+ "options": [
+ {
+ "name": "default",
+ "value": "true"
+ }
+ ]
+ },
+ {
+ "id": 4,
+ "name": "message",
+ "type": "string"
+ },
+ {
+ "id": 5,
+ "name": "status",
+ "type": "Status"
+ },
+ {
+ "id": 6,
+ "name": "getCertResponseProto",
+ "type": "SCMGetCertResponseProto"
+ }
+ ]
+ },
+ {
+ "name": "SCMGetDataNodeCertRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "datanodeDetails",
+ "type": "DatanodeDetailsProto"
+ },
+ {
+ "id": 2,
+ "name": "CSR",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "SCMGetOMCertRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "omDetails",
+ "type": "OzoneManagerDetailsProto"
+ },
+ {
+ "id": 2,
+ "name": "CSR",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "SCMGetCertificateRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "certSerialId",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "SCMGetCACertificateRequestProto"
+ },
+ {
+ "name": "SCMGetCertResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "responseCode",
+ "type": "ResponseCode"
+ },
+ {
+ "id": 2,
+ "name": "x509Certificate",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "x509CACertificate",
+ "type": "string"
+ }
+ ]
+ }
+ ],
+ "services": [
+ {
+ "name": "SCMSecurityProtocolService",
+ "rpcs": [
+ {
+ "name": "submitRequest",
+ "in_type": "SCMSecurityRequest",
+ "out_type": "SCMSecurityResponse"
+ }
+ ]
+ }
+ ],
+ "imports": [
+ {
+ "path": "hdds.proto"
+ }
+ ],
+ "package": {
+ "name": "hadoop.hdds.security"
+ },
+ "options": [
+ {
+ "name": "java_package",
+ "value": "org.apache.hadoop.hdds.protocol.proto"
+ },
+ {
+ "name": "java_outer_classname",
+ "value": "SCMSecurityProtocolProtos"
+ },
+ {
+ "name": "java_generic_services",
+ "value": "true"
+ },
+ {
+ "name": "java_generate_equals_and_hash",
+ "value": "true"
+ }
+ ]
+ }
+ },
+ {
+ "protopath": "ScmBlockLocationProtocol.proto",
+ "def": {
+ "enums": [
+ {
+ "name": "Type",
+ "enum_fields": [
+ {
+ "name": "AllocateScmBlock",
+ "integer": 11
+ },
+ {
+ "name": "DeleteScmKeyBlocks",
+ "integer": 12
+ },
+ {
+ "name": "GetScmInfo",
+ "integer": 13
+ },
+ {
+ "name": "SortDatanodes",
+ "integer": 14
+ }
+ ]
+ },
+ {
+ "name": "Status",
+ "enum_fields": [
+ {
+ "name": "OK",
+ "integer": 1
+ },
+ {
+ "name": "FAILED_TO_LOAD_NODEPOOL",
+ "integer": 2
+ },
+ {
+ "name": "FAILED_TO_FIND_NODE_IN_POOL",
+ "integer": 3
+ },
+ {
+ "name": "FAILED_TO_FIND_HEALTHY_NODES",
+ "integer": 4
+ },
+ {
+ "name": "FAILED_TO_FIND_NODES_WITH_SPACE",
+ "integer": 5
+ },
+ {
+ "name": "FAILED_TO_FIND_SUITABLE_NODE",
+ "integer": 6
+ },
+ {
+ "name": "INVALID_CAPACITY",
+ "integer": 7
+ },
+ {
+ "name": "INVALID_BLOCK_SIZE",
+ "integer": 8
+ },
+ {
+ "name": "SAFE_MODE_EXCEPTION",
+ "integer": 9
+ },
+ {
+ "name": "FAILED_TO_LOAD_OPEN_CONTAINER",
+ "integer": 10
+ },
+ {
+ "name": "FAILED_TO_ALLOCATE_CONTAINER",
+ "integer": 11
+ },
+ {
+ "name": "FAILED_TO_CHANGE_CONTAINER_STATE",
+ "integer": 12
+ },
+ {
+ "name": "FAILED_TO_CHANGE_PIPELINE_STATE",
+ "integer": 13
+ },
+ {
+ "name": "CONTAINER_EXISTS",
+ "integer": 14
+ },
+ {
+ "name": "FAILED_TO_FIND_CONTAINER",
+ "integer": 15
+ },
+ {
+ "name": "FAILED_TO_FIND_CONTAINER_WITH_SPACE",
+ "integer": 16
+ },
+ {
+ "name": "BLOCK_EXISTS",
+ "integer": 17
+ },
+ {
+ "name": "FAILED_TO_FIND_BLOCK",
+ "integer": 18
+ },
+ {
+ "name": "IO_EXCEPTION",
+ "integer": 19
+ },
+ {
+ "name": "UNEXPECTED_CONTAINER_STATE",
+ "integer": 20
+ },
+ {
+ "name": "SCM_NOT_INITIALIZED",
+ "integer": 21
+ },
+ {
+ "name": "DUPLICATE_DATANODE",
+ "integer": 22
+ },
+ {
+ "name": "NO_SUCH_DATANODE",
+ "integer": 23
+ },
+ {
+ "name": "NO_REPLICA_FOUND",
+ "integer": 24
+ },
+ {
+ "name": "FAILED_TO_FIND_ACTIVE_PIPELINE",
+ "integer": 25
+ },
+ {
+ "name": "FAILED_TO_INIT_CONTAINER_PLACEMENT_POLICY",
+ "integer": 26
+ },
+ {
+ "name": "FAILED_TO_ALLOCATE_ENOUGH_BLOCKS",
+ "integer": 27
+ },
+ {
+ "name": "INTERNAL_ERROR",
+ "integer": 29
+ }
+ ]
+ },
+ {
+ "name": "DeleteScmBlockResult.Result",
+ "enum_fields": [
+ {
+ "name": "success",
+ "integer": 1
+ },
+ {
+ "name": "safeMode",
+ "integer": 2
+ },
+ {
+ "name": "errorNotFound",
+ "integer": 3
+ },
+ {
+ "name": "unknownFailure",
+ "integer": 4
+ }
+ ]
+ }
+ ],
+ "messages": [
+ {
+ "name": "SCMBlockLocationRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "cmdType",
+ "type": "Type"
+ },
+ {
+ "id": 2,
+ "name": "traceID",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "userInfo",
+ "type": "UserInfo"
+ },
+ {
+ "id": 11,
+ "name": "allocateScmBlockRequest",
+ "type": "AllocateScmBlockRequestProto"
+ },
+ {
+ "id": 12,
+ "name": "deleteScmKeyBlocksRequest",
+ "type": "DeleteScmKeyBlocksRequestProto"
+ },
+ {
+ "id": 13,
+ "name": "getScmInfoRequest",
+ "type": "hadoop.hdds.GetScmInfoRequestProto"
+ },
+ {
+ "id": 14,
+ "name": "sortDatanodesRequest",
+ "type": "SortDatanodesRequestProto"
+ }
+ ]
+ },
+ {
+ "name": "SCMBlockLocationResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "cmdType",
+ "type": "Type"
+ },
+ {
+ "id": 2,
+ "name": "traceID",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "success",
+ "type": "bool",
+ "options": [
+ {
+ "name": "default",
+ "value": "true"
+ }
+ ]
+ },
+ {
+ "id": 4,
+ "name": "message",
+ "type": "string"
+ },
+ {
+ "id": 5,
+ "name": "status",
+ "type": "Status"
+ },
+ {
+ "id": 6,
+ "name": "leaderOMNodeId",
+ "type": "string"
+ },
+ {
+ "id": 11,
+ "name": "allocateScmBlockResponse",
+ "type": "AllocateScmBlockResponseProto"
+ },
+ {
+ "id": 12,
+ "name": "deleteScmKeyBlocksResponse",
+ "type": "DeleteScmKeyBlocksResponseProto"
+ },
+ {
+ "id": 13,
+ "name": "getScmInfoResponse",
+ "type": "hadoop.hdds.GetScmInfoResponseProto"
+ },
+ {
+ "id": 14,
+ "name": "sortDatanodesResponse",
+ "type": "SortDatanodesResponseProto"
+ }
+ ]
+ },
+ {
+ "name": "UserInfo",
+ "fields": [
+ {
+ "id": 1,
+ "name": "userName",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "remoteAddress",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "AllocateScmBlockRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "size",
+ "type": "uint64"
+ },
+ {
+ "id": 2,
+ "name": "numBlocks",
+ "type": "uint32"
+ },
+ {
+ "id": 3,
+ "name": "type",
+ "type": "ReplicationType"
+ },
+ {
+ "id": 4,
+ "name": "factor",
+ "type": "hadoop.hdds.ReplicationFactor"
+ },
+ {
+ "id": 5,
+ "name": "owner",
+ "type": "string"
+ },
+ {
+ "id": 7,
+ "name": "excludeList",
+ "type": "ExcludeListProto"
+ }
+ ]
+ },
+ {
+ "name": "DeleteScmKeyBlocksRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "keyBlocks",
+ "type": "KeyBlocks",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "KeyBlocks",
+ "fields": [
+ {
+ "id": 1,
+ "name": "key",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "blocks",
+ "type": "BlockID",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "DeleteScmKeyBlocksResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "results",
+ "type": "DeleteKeyBlocksResultProto",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "DeleteKeyBlocksResultProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "objectKey",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "blockResults",
+ "type": "DeleteScmBlockResult",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "DeleteScmBlockResult",
+ "fields": [
+ {
+ "id": 1,
+ "name": "result",
+ "type": "Result"
+ },
+ {
+ "id": 2,
+ "name": "blockID",
+ "type": "BlockID"
+ }
+ ]
+ },
+ {
+ "name": "AllocateBlockResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "containerBlockID",
+ "type": "ContainerBlockID"
+ },
+ {
+ "id": 2,
+ "name": "pipeline",
+ "type": "hadoop.hdds.Pipeline"
+ }
+ ]
+ },
+ {
+ "name": "AllocateScmBlockResponseProto",
+ "fields": [
+ {
+ "id": 3,
+ "name": "blocks",
+ "type": "AllocateBlockResponse",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "SortDatanodesRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "client",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "nodeNetworkName",
+ "type": "string",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "SortDatanodesResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "node",
+ "type": "DatanodeDetailsProto",
+ "is_repeated": true
+ }
+ ]
+ }
+ ],
+ "services": [
+ {
+ "name": "ScmBlockLocationProtocolService",
+ "rpcs": [
+ {
+ "name": "send",
+ "in_type": "SCMBlockLocationRequest",
+ "out_type": "SCMBlockLocationResponse"
+ }
+ ]
+ }
+ ],
+ "imports": [
+ {
+ "path": "hdds.proto"
+ }
+ ],
+ "package": {
+ "name": "hadoop.hdds.block"
+ },
+ "options": [
+ {
+ "name": "java_package",
+ "value": "org.apache.hadoop.hdds.protocol.proto"
+ },
+ {
+ "name": "java_outer_classname",
+ "value": "ScmBlockLocationProtocolProtos"
+ },
+ {
+ "name": "java_generic_services",
+ "value": "true"
+ },
+ {
+ "name": "java_generate_equals_and_hash",
+ "value": "true"
+ }
+ ]
+ }
+ },
+ {
+ "protopath": "Security.proto",
+ "def": {
+ "messages": [
+ {
+ "name": "TokenProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "identifier",
+ "type": "bytes"
+ },
+ {
+ "id": 2,
+ "name": "password",
+ "type": "bytes"
+ },
+ {
+ "id": 3,
+ "name": "kind",
+ "type": "string"
+ },
+ {
+ "id": 4,
+ "name": "service",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "CredentialsKVProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "alias",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "token",
+ "type": "hadoop.common.TokenProto"
+ },
+ {
+ "id": 3,
+ "name": "secret",
+ "type": "bytes"
+ }
+ ]
+ },
+ {
+ "name": "CredentialsProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "tokens",
+ "type": "hadoop.common.CredentialsKVProto",
+ "is_repeated": true
+ },
+ {
+ "id": 2,
+ "name": "secrets",
+ "type": "hadoop.common.CredentialsKVProto",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "GetDelegationTokenRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "renewer",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "GetDelegationTokenResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "token",
+ "type": "hadoop.common.TokenProto"
+ }
+ ]
+ },
+ {
+ "name": "RenewDelegationTokenRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "token",
+ "type": "hadoop.common.TokenProto"
+ }
+ ]
+ },
+ {
+ "name": "RenewDelegationTokenResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "newExpiryTime",
+ "type": "uint64"
+ }
+ ]
+ },
+ {
+ "name": "CancelDelegationTokenRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "token",
+ "type": "hadoop.common.TokenProto"
+ }
+ ]
+ },
+ {
+ "name": "CancelDelegationTokenResponseProto"
+ }
+ ],
+ "package": {
+ "name": "hadoop.common"
+ },
+ "options": [
+ {
+ "name": "java_package",
+ "value": "org.apache.hadoop.security.proto"
+ },
+ {
+ "name": "java_outer_classname",
+ "value": "SecurityProtos"
+ },
+ {
+ "name": "java_generic_services",
+ "value": "true"
+ },
+ {
+ "name": "java_generate_equals_and_hash",
+ "value": "true"
+ }
+ ]
+ }
+ },
+ {
+ "protopath": "StorageContainerLocationProtocol.proto",
+ "def": {
+ "enums": [
+ {
+ "name": "ScmContainerLocationResponse.Status",
+ "enum_fields": [
+ {
+ "name": "OK",
+ "integer": 1
+ },
+ {
+ "name": "CONTAINER_ALREADY_EXISTS",
+ "integer": 2
+ },
+ {
+ "name": "CONTAINER_IS_MISSING",
+ "integer": 3
+ }
+ ]
+ },
+ {
+ "name": "Type",
+ "enum_fields": [
+ {
+ "name": "AllocateContainer",
+ "integer": 1
+ },
+ {
+ "name": "GetContainer",
+ "integer": 2
+ },
+ {
+ "name": "GetContainerWithPipeline",
+ "integer": 3
+ },
+ {
+ "name": "ListContainer",
+ "integer": 4
+ },
+ {
+ "name": "DeleteContainer",
+ "integer": 5
+ },
+ {
+ "name": "QueryNode",
+ "integer": 6
+ },
+ {
+ "name": "CloseContainer",
+ "integer": 7
+ },
+ {
+ "name": "AllocatePipeline",
+ "integer": 8
+ },
+ {
+ "name": "ListPipelines",
+ "integer": 9
+ },
+ {
+ "name": "ActivatePipeline",
+ "integer": 10
+ },
+ {
+ "name": "DeactivatePipeline",
+ "integer": 11
+ },
+ {
+ "name": "ClosePipeline",
+ "integer": 12
+ },
+ {
+ "name": "GetScmInfo",
+ "integer": 13
+ },
+ {
+ "name": "InSafeMode",
+ "integer": 14
+ },
+ {
+ "name": "ForceExitSafeMode",
+ "integer": 15
+ },
+ {
+ "name": "StartReplicationManager",
+ "integer": 16
+ },
+ {
+ "name": "StopReplicationManager",
+ "integer": 17
+ },
+ {
+ "name": "GetReplicationManagerStatus",
+ "integer": 18
+ },
+ {
+ "name": "GetPipeline",
+ "integer": 19
+ },
+ {
+ "name": "GetContainerWithPipelineBatch",
+ "integer": 20
+ }
+ ]
+ },
+ {
+ "name": "ContainerResponseProto.Error",
+ "enum_fields": [
+ {
+ "name": "success",
+ "integer": 1
+ },
+ {
+ "name": "errorContainerAlreadyExists",
+ "integer": 2
+ },
+ {
+ "name": "errorContainerMissing",
+ "integer": 3
+ }
+ ]
+ },
+ {
+ "name": "PipelineResponseProto.Error",
+ "enum_fields": [
+ {
+ "name": "success",
+ "integer": 1
+ },
+ {
+ "name": "errorPipelineAlreadyExists",
+ "integer": 2
+ }
+ ]
+ }
+ ],
+ "messages": [
+ {
+ "name": "ScmContainerLocationRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "cmdType",
+ "type": "Type"
+ },
+ {
+ "id": 2,
+ "name": "traceID",
+ "type": "string"
+ },
+ {
+ "id": 6,
+ "name": "containerRequest",
+ "type": "ContainerRequestProto"
+ },
+ {
+ "id": 7,
+ "name": "getContainerRequest",
+ "type": "GetContainerRequestProto"
+ },
+ {
+ "id": 8,
+ "name": "getContainerWithPipelineRequest",
+ "type": "GetContainerWithPipelineRequestProto"
+ },
+ {
+ "id": 9,
+ "name": "scmListContainerRequest",
+ "type": "SCMListContainerRequestProto"
+ },
+ {
+ "id": 10,
+ "name": "scmDeleteContainerRequest",
+ "type": "SCMDeleteContainerRequestProto"
+ },
+ {
+ "id": 11,
+ "name": "nodeQueryRequest",
+ "type": "NodeQueryRequestProto"
+ },
+ {
+ "id": 12,
+ "name": "scmCloseContainerRequest",
+ "type": "SCMCloseContainerRequestProto"
+ },
+ {
+ "id": 13,
+ "name": "pipelineRequest",
+ "type": "PipelineRequestProto"
+ },
+ {
+ "id": 14,
+ "name": "listPipelineRequest",
+ "type": "ListPipelineRequestProto"
+ },
+ {
+ "id": 15,
+ "name": "activatePipelineRequest",
+ "type": "ActivatePipelineRequestProto"
+ },
+ {
+ "id": 16,
+ "name": "deactivatePipelineRequest",
+ "type": "DeactivatePipelineRequestProto"
+ },
+ {
+ "id": 17,
+ "name": "closePipelineRequest",
+ "type": "ClosePipelineRequestProto"
+ },
+ {
+ "id": 18,
+ "name": "getScmInfoRequest",
+ "type": "GetScmInfoRequestProto"
+ },
+ {
+ "id": 19,
+ "name": "inSafeModeRequest",
+ "type": "InSafeModeRequestProto"
+ },
+ {
+ "id": 20,
+ "name": "forceExitSafeModeRequest",
+ "type": "ForceExitSafeModeRequestProto"
+ },
+ {
+ "id": 21,
+ "name": "startReplicationManagerRequest",
+ "type": "StartReplicationManagerRequestProto"
+ },
+ {
+ "id": 22,
+ "name": "stopReplicationManagerRequest",
+ "type": "StopReplicationManagerRequestProto"
+ },
+ {
+ "id": 23,
+ "name": "seplicationManagerStatusRequest",
+ "type": "ReplicationManagerStatusRequestProto"
+ },
+ {
+ "id": 24,
+ "name": "getPipelineRequest",
+ "type": "GetPipelineRequestProto"
+ },
+ {
+ "id": 25,
+ "name": "getContainerWithPipelineBatchRequest",
+ "type": "GetContainerWithPipelineBatchRequestProto"
+ }
+ ]
+ },
+ {
+ "name": "ScmContainerLocationResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "cmdType",
+ "type": "Type"
+ },
+ {
+ "id": 2,
+ "name": "traceID",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "success",
+ "type": "bool",
+ "options": [
+ {
+ "name": "default",
+ "value": "true"
+ }
+ ]
+ },
+ {
+ "id": 4,
+ "name": "message",
+ "type": "string"
+ },
+ {
+ "id": 5,
+ "name": "status",
+ "type": "Status"
+ },
+ {
+ "id": 6,
+ "name": "containerResponse",
+ "type": "ContainerResponseProto"
+ },
+ {
+ "id": 7,
+ "name": "getContainerResponse",
+ "type": "GetContainerResponseProto"
+ },
+ {
+ "id": 8,
+ "name": "getContainerWithPipelineResponse",
+ "type": "GetContainerWithPipelineResponseProto"
+ },
+ {
+ "id": 9,
+ "name": "scmListContainerResponse",
+ "type": "SCMListContainerResponseProto"
+ },
+ {
+ "id": 10,
+ "name": "scmDeleteContainerResponse",
+ "type": "SCMDeleteContainerResponseProto"
+ },
+ {
+ "id": 11,
+ "name": "nodeQueryResponse",
+ "type": "NodeQueryResponseProto"
+ },
+ {
+ "id": 12,
+ "name": "scmCloseContainerResponse",
+ "type": "SCMCloseContainerResponseProto"
+ },
+ {
+ "id": 13,
+ "name": "pipelineResponse",
+ "type": "PipelineResponseProto"
+ },
+ {
+ "id": 14,
+ "name": "listPipelineResponse",
+ "type": "ListPipelineResponseProto"
+ },
+ {
+ "id": 15,
+ "name": "activatePipelineResponse",
+ "type": "ActivatePipelineResponseProto"
+ },
+ {
+ "id": 16,
+ "name": "deactivatePipelineResponse",
+ "type": "DeactivatePipelineResponseProto"
+ },
+ {
+ "id": 17,
+ "name": "closePipelineResponse",
+ "type": "ClosePipelineResponseProto"
+ },
+ {
+ "id": 18,
+ "name": "getScmInfoResponse",
+ "type": "GetScmInfoResponseProto"
+ },
+ {
+ "id": 19,
+ "name": "inSafeModeResponse",
+ "type": "InSafeModeResponseProto"
+ },
+ {
+ "id": 20,
+ "name": "forceExitSafeModeResponse",
+ "type": "ForceExitSafeModeResponseProto"
+ },
+ {
+ "id": 21,
+ "name": "startReplicationManagerResponse",
+ "type": "StartReplicationManagerResponseProto"
+ },
+ {
+ "id": 22,
+ "name": "stopReplicationManagerResponse",
+ "type": "StopReplicationManagerResponseProto"
+ },
+ {
+ "id": 23,
+ "name": "replicationManagerStatusResponse",
+ "type": "ReplicationManagerStatusResponseProto"
+ },
+ {
+ "id": 24,
+ "name": "getPipelineResponse",
+ "type": "GetPipelineResponseProto"
+ },
+ {
+ "id": 25,
+ "name": "getContainerWithPipelineBatchResponse",
+ "type": "GetContainerWithPipelineBatchResponseProto"
+ }
+ ]
+ },
+ {
+ "name": "ContainerRequestProto",
+ "fields": [
+ {
+ "id": 2,
+ "name": "replicationFactor",
+ "type": "ReplicationFactor"
+ },
+ {
+ "id": 3,
+ "name": "replicationType",
+ "type": "ReplicationType"
+ },
+ {
+ "id": 4,
+ "name": "owner",
+ "type": "string"
+ },
+ {
+ "id": 5,
+ "name": "traceID",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "ContainerResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "errorCode",
+ "type": "Error"
+ },
+ {
+ "id": 2,
+ "name": "containerWithPipeline",
+ "type": "ContainerWithPipeline"
+ },
+ {
+ "id": 3,
+ "name": "errorMessage",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "GetContainerRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "containerID",
+ "type": "int64"
+ },
+ {
+ "id": 2,
+ "name": "traceID",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "GetContainerResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "containerInfo",
+ "type": "ContainerInfoProto"
+ }
+ ]
+ },
+ {
+ "name": "GetContainerWithPipelineRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "containerID",
+ "type": "int64"
+ },
+ {
+ "id": 2,
+ "name": "traceID",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "GetContainerWithPipelineResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "containerWithPipeline",
+ "type": "ContainerWithPipeline"
+ }
+ ]
+ },
+ {
+ "name": "GetContainerWithPipelineBatchRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "containerIDs",
+ "type": "int64",
+ "is_repeated": true
+ },
+ {
+ "id": 2,
+ "name": "traceID",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "GetContainerWithPipelineBatchResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "containerWithPipelines",
+ "type": "ContainerWithPipeline",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "SCMListContainerRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "count",
+ "type": "uint32"
+ },
+ {
+ "id": 2,
+ "name": "startContainerID",
+ "type": "uint64"
+ },
+ {
+ "id": 3,
+ "name": "traceID",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "SCMListContainerResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "containers",
+ "type": "ContainerInfoProto",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "SCMDeleteContainerRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "containerID",
+ "type": "int64"
+ },
+ {
+ "id": 2,
+ "name": "traceID",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "SCMDeleteContainerResponseProto"
+ },
+ {
+ "name": "SCMCloseContainerRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "containerID",
+ "type": "int64"
+ },
+ {
+ "id": 2,
+ "name": "traceID",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "SCMCloseContainerResponseProto"
+ },
+ {
+ "name": "NodeQueryRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "state",
+ "type": "NodeState"
+ },
+ {
+ "id": 2,
+ "name": "scope",
+ "type": "QueryScope"
+ },
+ {
+ "id": 3,
+ "name": "poolName",
+ "type": "string"
+ },
+ {
+ "id": 4,
+ "name": "traceID",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "NodeQueryResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "datanodes",
+ "type": "Node",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "PipelineRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "replicationType",
+ "type": "ReplicationType"
+ },
+ {
+ "id": 2,
+ "name": "replicationFactor",
+ "type": "ReplicationFactor"
+ },
+ {
+ "id": 3,
+ "name": "nodePool",
+ "type": "NodePool"
+ },
+ {
+ "id": 4,
+ "name": "pipelineID",
+ "type": "string"
+ },
+ {
+ "id": 5,
+ "name": "traceID",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "PipelineResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "errorCode",
+ "type": "Error"
+ },
+ {
+ "id": 2,
+ "name": "pipeline",
+ "type": "Pipeline"
+ },
+ {
+ "id": 3,
+ "name": "errorMessage",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "ListPipelineRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "traceID",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "ListPipelineResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "pipelines",
+ "type": "Pipeline",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "GetPipelineRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "pipelineID",
+ "type": "PipelineID"
+ },
+ {
+ "id": 2,
+ "name": "traceID",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "GetPipelineResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "pipeline",
+ "type": "Pipeline"
+ }
+ ]
+ },
+ {
+ "name": "ActivatePipelineRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "pipelineID",
+ "type": "PipelineID"
+ },
+ {
+ "id": 2,
+ "name": "traceID",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "ActivatePipelineResponseProto"
+ },
+ {
+ "name": "DeactivatePipelineRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "pipelineID",
+ "type": "PipelineID"
+ },
+ {
+ "id": 2,
+ "name": "traceID",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "DeactivatePipelineResponseProto"
+ },
+ {
+ "name": "ClosePipelineRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "pipelineID",
+ "type": "PipelineID"
+ },
+ {
+ "id": 2,
+ "name": "traceID",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "ClosePipelineResponseProto"
+ },
+ {
+ "name": "InSafeModeRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "traceID",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "InSafeModeResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "inSafeMode",
+ "type": "bool"
+ }
+ ]
+ },
+ {
+ "name": "ForceExitSafeModeRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "traceID",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "ForceExitSafeModeResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "exitedSafeMode",
+ "type": "bool"
+ }
+ ]
+ },
+ {
+ "name": "StartReplicationManagerRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "traceID",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "StartReplicationManagerResponseProto"
+ },
+ {
+ "name": "StopReplicationManagerRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "traceID",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "StopReplicationManagerResponseProto"
+ },
+ {
+ "name": "ReplicationManagerStatusRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "traceID",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "ReplicationManagerStatusResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "isRunning",
+ "type": "bool"
+ }
+ ]
+ }
+ ],
+ "services": [
+ {
+ "name": "StorageContainerLocationProtocolService",
+ "rpcs": [
+ {
+ "name": "submitRequest",
+ "in_type": "ScmContainerLocationRequest",
+ "out_type": "ScmContainerLocationResponse"
+ }
+ ]
+ }
+ ],
+ "imports": [
+ {
+ "path": "hdds.proto"
+ }
+ ],
+ "package": {
+ "name": "hadoop.hdds.container"
+ },
+ "options": [
+ {
+ "name": "java_package",
+ "value": "org.apache.hadoop.hdds.protocol.proto"
+ },
+ {
+ "name": "java_outer_classname",
+ "value": "StorageContainerLocationProtocolProtos"
+ },
+ {
+ "name": "java_generic_services",
+ "value": "true"
+ },
+ {
+ "name": "java_generate_equals_and_hash",
+ "value": "true"
+ }
+ ]
+ }
+ },
+ {
+ "protopath": "hdds.proto",
+ "def": {
+ "enums": [
+ {
+ "name": "PipelineState",
+ "enum_fields": [
+ {
+ "name": "PIPELINE_ALLOCATED",
+ "integer": 1
+ },
+ {
+ "name": "PIPELINE_OPEN",
+ "integer": 2
+ },
+ {
+ "name": "PIPELINE_DORMANT",
+ "integer": 3
+ },
+ {
+ "name": "PIPELINE_CLOSED",
+ "integer": 4
+ }
+ ]
+ },
+ {
+ "name": "NodeType",
+ "enum_fields": [
+ {
+ "name": "OM",
+ "integer": 1
+ },
+ {
+ "name": "SCM",
+ "integer": 2
+ },
+ {
+ "name": "DATANODE",
+ "integer": 3
+ },
+ {
+ "name": "RECON",
+ "integer": 4
+ }
+ ]
+ },
+ {
+ "name": "NodeState",
+ "enum_fields": [
+ {
+ "name": "HEALTHY",
+ "integer": 1
+ },
+ {
+ "name": "STALE",
+ "integer": 2
+ },
+ {
+ "name": "DEAD",
+ "integer": 3
+ },
+ {
+ "name": "DECOMMISSIONING",
+ "integer": 4
+ },
+ {
+ "name": "DECOMMISSIONED",
+ "integer": 5
+ }
+ ]
+ },
+ {
+ "name": "QueryScope",
+ "enum_fields": [
+ {
+ "name": "CLUSTER",
+ "integer": 1
+ },
+ {
+ "name": "POOL",
+ "integer": 2
+ }
+ ]
+ },
+ {
+ "name": "LifeCycleState",
+ "enum_fields": [
+ {
+ "name": "OPEN",
+ "integer": 1
+ },
+ {
+ "name": "CLOSING",
+ "integer": 2
+ },
+ {
+ "name": "QUASI_CLOSED",
+ "integer": 3
+ },
+ {
+ "name": "CLOSED",
+ "integer": 4
+ },
+ {
+ "name": "DELETING",
+ "integer": 5
+ },
+ {
+ "name": "DELETED",
+ "integer": 6
+ }
+ ]
+ },
+ {
+ "name": "LifeCycleEvent",
+ "enum_fields": [
+ {
+ "name": "FINALIZE",
+ "integer": 1
+ },
+ {
+ "name": "QUASI_CLOSE",
+ "integer": 2
+ },
+ {
+ "name": "CLOSE",
+ "integer": 3
+ },
+ {
+ "name": "FORCE_CLOSE",
+ "integer": 4
+ },
+ {
+ "name": "DELETE",
+ "integer": 5
+ },
+ {
+ "name": "CLEANUP",
+ "integer": 6
+ }
+ ]
+ },
+ {
+ "name": "ReplicationType",
+ "enum_fields": [
+ {
+ "name": "RATIS",
+ "integer": 1
+ },
+ {
+ "name": "STAND_ALONE",
+ "integer": 2
+ },
+ {
+ "name": "CHAINED",
+ "integer": 3
+ }
+ ]
+ },
+ {
+ "name": "ReplicationFactor",
+ "enum_fields": [
+ {
+ "name": "ONE",
+ "integer": 1
+ },
+ {
+ "name": "THREE",
+ "integer": 3
+ }
+ ]
+ },
+ {
+ "name": "ScmOps",
+ "enum_fields": [
+ {
+ "name": "allocateBlock",
+ "integer": 1
+ },
+ {
+ "name": "keyBlocksInfoList",
+ "integer": 2
+ },
+ {
+ "name": "getScmInfo",
+ "integer": 3
+ },
+ {
+ "name": "deleteBlock",
+ "integer": 4
+ },
+ {
+ "name": "createReplicationPipeline",
+ "integer": 5
+ },
+ {
+ "name": "allocateContainer",
+ "integer": 6
+ },
+ {
+ "name": "getContainer",
+ "integer": 7
+ },
+ {
+ "name": "getContainerWithPipeline",
+ "integer": 8
+ },
+ {
+ "name": "listContainer",
+ "integer": 9
+ },
+ {
+ "name": "deleteContainer",
+ "integer": 10
+ },
+ {
+ "name": "queryNode",
+ "integer": 11
+ }
+ ]
+ },
+ {
+ "name": "BlockTokenSecretProto.AccessModeProto",
+ "enum_fields": [
+ {
+ "name": "READ",
+ "integer": 1
+ },
+ {
+ "name": "WRITE",
+ "integer": 2
+ },
+ {
+ "name": "COPY",
+ "integer": 3
+ },
+ {
+ "name": "DELETE",
+ "integer": 4
+ }
+ ]
+ }
+ ],
+ "messages": [
+ {
+ "name": "DatanodeDetailsProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "uuid",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "ipAddress",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "hostName",
+ "type": "string"
+ },
+ {
+ "id": 4,
+ "name": "ports",
+ "type": "Port",
+ "is_repeated": true
+ },
+ {
+ "id": 5,
+ "name": "certSerialId",
+ "type": "string"
+ },
+ {
+ "id": 6,
+ "name": "networkName",
+ "type": "string"
+ },
+ {
+ "id": 7,
+ "name": "networkLocation",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "OzoneManagerDetailsProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "uuid",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "ipAddress",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "hostName",
+ "type": "string"
+ },
+ {
+ "id": 4,
+ "name": "ports",
+ "type": "Port",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "Port",
+ "fields": [
+ {
+ "id": 1,
+ "name": "name",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "value",
+ "type": "uint32"
+ }
+ ]
+ },
+ {
+ "name": "PipelineID",
+ "fields": [
+ {
+ "id": 1,
+ "name": "id",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "Pipeline",
+ "fields": [
+ {
+ "id": 1,
+ "name": "members",
+ "type": "DatanodeDetailsProto",
+ "is_repeated": true
+ },
+ {
+ "id": 2,
+ "name": "state",
+ "type": "PipelineState",
+ "options": [
+ {
+ "name": "default",
+ "value": "PIPELINE_ALLOCATED"
+ }
+ ]
+ },
+ {
+ "id": 3,
+ "name": "type",
+ "type": "ReplicationType",
+ "options": [
+ {
+ "name": "default",
+ "value": "STAND_ALONE"
+ }
+ ]
+ },
+ {
+ "id": 4,
+ "name": "factor",
+ "type": "ReplicationFactor",
+ "options": [
+ {
+ "name": "default",
+ "value": "ONE"
+ }
+ ]
+ },
+ {
+ "id": 5,
+ "name": "id",
+ "type": "PipelineID"
+ },
+ {
+ "id": 6,
+ "name": "leaderID",
+ "type": "string"
+ },
+ {
+ "id": 7,
+ "name": "memberOrders",
+ "type": "uint32",
+ "is_repeated": true
+ },
+ {
+ "id": 8,
+ "name": "creationTimeStamp",
+ "type": "uint64"
+ }
+ ]
+ },
+ {
+ "name": "KeyValue",
+ "fields": [
+ {
+ "id": 1,
+ "name": "key",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "value",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "Node",
+ "fields": [
+ {
+ "id": 1,
+ "name": "nodeID",
+ "type": "DatanodeDetailsProto"
+ },
+ {
+ "id": 2,
+ "name": "nodeStates",
+ "type": "NodeState",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "NodePool",
+ "fields": [
+ {
+ "id": 1,
+ "name": "nodes",
+ "type": "Node",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "ContainerInfoProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "containerID",
+ "type": "int64"
+ },
+ {
+ "id": 2,
+ "name": "state",
+ "type": "LifeCycleState"
+ },
+ {
+ "id": 3,
+ "name": "pipelineID",
+ "type": "PipelineID"
+ },
+ {
+ "id": 4,
+ "name": "usedBytes",
+ "type": "uint64"
+ },
+ {
+ "id": 5,
+ "name": "numberOfKeys",
+ "type": "uint64"
+ },
+ {
+ "id": 6,
+ "name": "stateEnterTime",
+ "type": "int64"
+ },
+ {
+ "id": 7,
+ "name": "owner",
+ "type": "string"
+ },
+ {
+ "id": 8,
+ "name": "deleteTransactionId",
+ "type": "int64"
+ },
+ {
+ "id": 9,
+ "name": "sequenceId",
+ "type": "int64"
+ },
+ {
+ "id": 10,
+ "name": "replicationFactor",
+ "type": "ReplicationFactor"
+ },
+ {
+ "id": 11,
+ "name": "replicationType",
+ "type": "ReplicationType"
+ }
+ ]
+ },
+ {
+ "name": "ContainerWithPipeline",
+ "fields": [
+ {
+ "id": 1,
+ "name": "containerInfo",
+ "type": "ContainerInfoProto"
+ },
+ {
+ "id": 2,
+ "name": "pipeline",
+ "type": "Pipeline"
+ }
+ ]
+ },
+ {
+ "name": "GetScmInfoRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "traceID",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "GetScmInfoResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "clusterId",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "scmId",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "ExcludeListProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "datanodes",
+ "type": "string",
+ "is_repeated": true
+ },
+ {
+ "id": 2,
+ "name": "containerIds",
+ "type": "int64",
+ "is_repeated": true
+ },
+ {
+ "id": 3,
+ "name": "pipelineIds",
+ "type": "PipelineID",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "ContainerBlockID",
+ "fields": [
+ {
+ "id": 1,
+ "name": "containerID",
+ "type": "int64"
+ },
+ {
+ "id": 2,
+ "name": "localID",
+ "type": "int64"
+ }
+ ]
+ },
+ {
+ "name": "BlockTokenSecretProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "ownerId",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "blockId",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "expiryDate",
+ "type": "uint64"
+ },
+ {
+ "id": 4,
+ "name": "omCertSerialId",
+ "type": "string"
+ },
+ {
+ "id": 5,
+ "name": "modes",
+ "type": "AccessModeProto",
+ "is_repeated": true
+ },
+ {
+ "id": 6,
+ "name": "maxLength",
+ "type": "uint64"
+ }
+ ]
+ },
+ {
+ "name": "BlockID",
+ "fields": [
+ {
+ "id": 1,
+ "name": "containerBlockID",
+ "type": "ContainerBlockID"
+ },
+ {
+ "id": 2,
+ "name": "blockCommitSequenceId",
+ "type": "uint64",
+ "options": [
+ {
+ "name": "default",
+ "value": "0"
+ }
+ ]
+ }
+ ]
+ }
+ ],
+ "package": {
+ "name": "hadoop.hdds"
+ },
+ "options": [
+ {
+ "name": "java_package",
+ "value": "org.apache.hadoop.hdds.protocol.proto"
+ },
+ {
+ "name": "java_outer_classname",
+ "value": "HddsProtos"
+ },
+ {
+ "name": "java_generic_services",
+ "value": "true"
+ },
+ {
+ "name": "java_generate_equals_and_hash",
+ "value": "true"
+ }
+ ]
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 5972cf0..5b4b931 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -197,10 +197,12 @@
</property>
<property>
<name>dfs.container.ratis.num.write.chunk.threads</name>
- <value>60</value>
+ <value>10</value>
<tag>OZONE, RATIS, PERFORMANCE</tag>
- <description>Maximum number of threads in the thread pool that Ratis
- will use for writing chunks (60 by default).
+ <description>Maximum number of threads in the thread pool that Datanode
+ will use for writing replicated chunks.
+ This is a per configured locations!
+ (10 thread per disk by default).
</description>
</property>
<property>
@@ -387,10 +389,20 @@
</description>
</property>
<property>
+ <name>ozone.client.stream.buffer.flush.delay</name>
+ <value>false</value>
+ <tag>OZONE, CLIENT</tag>
+ <description>If set true, when call flush() and determine whether the
+ data in the current buffer is greater than ozone.client.stream.buffer.size.
+ if greater than then send buffer to the datanode.
+ </description>
+ </property>
+ <property>
<name>ozone.client.stream.buffer.size</name>
<value>4MB</value>
<tag>OZONE, CLIENT</tag>
- <description>The size of chunks the client will send to the server.</description>
+ <description>The size of chunks the client will send to the server.
+ </description>
</property>
<property>
<name>ozone.client.stream.buffer.flush.size</name>
@@ -584,6 +596,16 @@
</description>
</property>
<property>
+ <name>ozone.om.volume.listall.allowed</name>
+ <value>true</value>
+ <tag>OM, MANAGEMENT</tag>
+ <description>
+ Allows everyone to list all volumes when set to true. Defaults to true.
+ When set to false, non-admin users can only list the volumes they have
+ access to. Admins can always list all volumes.
+ </description>
+ </property>
+ <property>
<name>ozone.om.user.max.volume</name>
<value>1024</value>
<tag>OM, MANAGEMENT</tag>
@@ -1413,15 +1435,13 @@
</property>
<property>
- <name>ozone.s3g.authentication.kerberos.principal</name>
- <value/>
+ <name>ozone.s3g.volume.name</name>
+ <value>s3v</value>
<tag>OZONE, S3GATEWAY</tag>
- <description>The server principal used by Ozone S3Gateway server. This is
- typically set to
- HTTP/_HOST@REALM.TLD The SPNEGO server principal begins with the prefix
- HTTP/ by convention.</description>
+ <description>
+ The volume name to access through the s3gateway.
+ </description>
</property>
-
<property>
<name>ozone.s3g.domain.name</name>
<value/>
@@ -1478,7 +1498,17 @@
</property>
<property>
- <name>ozone.s3g.keytab.file</name>
+ <name>ozone.s3g.http.auth.kerberos.principal</name>
+ <value/>
+ <tag>OZONE, S3GATEWAY</tag>
+ <description>The server principal used by Ozone S3Gateway server. This is
+ typically set to
+ HTTP/_HOST@REALM.TLD The SPNEGO server principal begins with the prefix
+ HTTP/ by convention.</description>
+ </property>
+
+ <property>
+ <name>ozone.s3g.http.auth.kerberos.keytab</name>
<value/>
<tag>OZONE, S3GATEWAY</tag>
<description>The keytab file used by the S3Gateway server to login as its
@@ -1497,11 +1527,21 @@
<property>
<name>ozone.security.enabled</name>
<value>false</value>
- <tag> OZONE, SECURITY</tag>
+ <tag>OZONE, SECURITY</tag>
<description>True if security is enabled for ozone. When this property is
true, hadoop.security.authentication should be Kerberos.
</description>
</property>
+ <property>
+ <name>ozone.security.http.kerberos.enabled</name>
+ <value>false</value>
+ <tag>OZONE, SECURITY</tag>
+ <description>True if Kerberos authentication for Ozone HTTP web consoles
+ is enabled using the SPNEGO protocol. When this property is
+ true, hadoop.security.authentication should be Kerberos and
+ ozone.security.enabled should be set to true.
+ </description>
+ </property>
<property>
<name>ozone.client.checksum.type</name>
@@ -1686,17 +1726,6 @@
</property>
<property>
- <name>ipc.client.rpc-timeout.ms</name>
- <value>900000</value>
- <description>
- RpcClient timeout on waiting response from server. The default value is
- set to 15 minutes. If ipc.client.ping is set to true and this rpc-timeout
- is greater than the value of ipc.ping.interval, the effective value of
- the rpc-timeout is rounded up to multiple of ipc.ping.interval.
- </description>
- </property>
-
- <property>
<name>ozone.om.ratis.snapshot.dir</name>
<value/>
<tag>OZONE, OM, STORAGE, MANAGEMENT, RATIS</tag>
@@ -1764,14 +1793,14 @@
<description>The OzoneManager service principal. Ex om/_HOST@REALM.COM</description>
</property>
<property>
- <name>ozone.om.http.kerberos.principal</name>
+ <name>ozone.om.http.auth.kerberos.principal</name>
<value>HTTP/_HOST@EXAMPLE.COM</value>
<description>
OzoneManager http server kerberos principal.
</description>
</property>
<property>
- <name>ozone.om.http.kerberos.keytab</name>
+ <name>ozone.om.http.auth.kerberos.keytab</name>
<value>/etc/security/keytabs/HTTP.keytab</value>
<description>
OzoneManager http server kerberos keytab.
@@ -2003,7 +2032,7 @@
</description>
</property>
<property>
- <name>ozone.freon.http.kerberos.principal</name>
+ <name>ozone.freon.http.auth.kerberos.principal</name>
<value>HTTP/_HOST@EXAMPLE.COM</value>
<tag>SECURITY</tag>
<description>
@@ -2011,7 +2040,7 @@
</description>
</property>
<property>
- <name>ozone.freon.http.kerberos.keytab</name>
+ <name>ozone.freon.http.auth.kerberos.keytab</name>
<value>/etc/security/keytabs/HTTP.keytab</value>
<tag>SECURITY</tag>
<description>
@@ -2065,7 +2094,7 @@
</property>
<property>
- <name>hdds.datanode.http.kerberos.principal</name>
+ <name>hdds.datanode.http.auth.kerberos.principal</name>
<value>HTTP/_HOST@EXAMPLE.COM</value>
<tag>HDDS, SECURITY, MANAGEMENT</tag>
<description>
@@ -2073,7 +2102,7 @@
</description>
</property>
<property>
- <name>hdds.datanode.http.kerberos.keytab</name>
+ <name>hdds.datanode.http.auth.kerberos.keytab</name>
<value>/etc/security/keytabs/HTTP.keytab</value>
<tag>HDDS, SECURITY, MANAGEMENT</tag>
<description>
@@ -2139,29 +2168,13 @@
</description>
</property>
<property>
- <name>ozone.client.failover.sleep.base.millis</name>
- <value>500</value>
+ <name>ozone.client.wait.between.retries.millis</name>
+ <value>2000</value>
<description>
- Expert only. The time to wait, in milliseconds, between failover
- attempts increases exponentially as a function of the number of
- attempts made so far, with a random factor of +/- 50%. This option
- specifies the base value used in the failover calculation. The
- first failover will retry immediately. The 2nd failover attempt
- will delay at least ozone.client.failover.sleep.base.millis
- milliseconds. And so on.
- </description>
- </property>
- <property>
- <name>ozone.client.failover.sleep.max.millis</name>
- <value>15000</value>
- <description>
- Expert only. The time to wait, in milliseconds, between failover
- attempts increases exponentially as a function of the number of
- attempts made so far, with a random factor of +/- 50%. This option
- specifies the maximum value to wait between failovers.
- Specifically, the time between two failover attempts will not
- exceed +/- 50% of ozone.client.failover.sleep.max.millis
- milliseconds.
+ Expert only. The time to wait, in milliseconds, between retry attempts
+ to contact OM. Wait time increases linearly if same OM is retried
+ again. If retrying on multiple OMs proxies in round robin fashion, the
+ wait time is introduced after all the OM proxies have been attempted once.
</description>
</property>
<property>
@@ -2215,16 +2228,7 @@
</description>
</property>
<property>
- <name>ozone.recon.keytab.file</name>
- <value/>
- <tag>RECON, SECURITY</tag>
- <description>
- DEPRECATED. Use "ozone.recon.http.kerberos.keytab.file" instead.
- The keytab file for HTTP Kerberos authentication in Recon.
- </description>
- </property>
- <property>
- <name>ozone.recon.http.kerberos.keytab.file</name>
+ <name>ozone.recon.http.auth.kerberos.keytab</name>
<value/>
<tag>RECON, SECURITY</tag>
<description>
@@ -2232,7 +2236,7 @@
</description>
</property>
<property>
- <name>ozone.recon.authentication.kerberos.principal</name>
+ <name>ozone.recon.http.auth.kerberos.principal</name>
<value/>
<tag>RECON</tag>
<description>The server principal used by Ozone Recon server. This is
@@ -2241,6 +2245,46 @@
</description>
</property>
<property>
+ <name>hdds.datanode.http.auth.type </name>
+ <value>simple</value>
+ <tag>DATANODE, SECURITY</tag>
+ <description> simple or kerberos. If kerberos is set, Kerberos SPNEOGO
+ will be used for http authentication.
+ </description>
+ </property>
+ <property>
+ <name>ozone.freon.http.auth.type </name>
+ <value>simple</value>
+ <tag>FREON, SECURITY</tag>
+ <description> simple or kerberos. If kerberos is set, Kerberos SPNEOGO
+ will be used for http authentication.
+ </description>
+ </property>
+ <property>
+ <name>ozone.om.http.auth.type </name>
+ <value>simple</value>
+ <tag>OM, SECURITY</tag>
+ <description> simple or kerberos. If kerberos is set, Kerberos SPNEOGO
+ will be used for http authentication.
+ </description>
+ </property>
+ <property>
+ <name>ozone.recon.http.auth.type </name>
+ <value>simple</value>
+ <tag>RECON, SECURITY</tag>
+ <description> simple or kerberos. If kerberos is set, Kerberos SPNEOGO
+ will be used for http authentication.
+ </description>
+ </property>
+ <property>
+ <name>ozone.s3g.http.auth.type </name>
+ <value>simple</value>
+ <tag>S3G, SECURITY</tag>
+ <description> simple or kerberos. If kerberos is set, Kerberos SPNEOGO
+ will be used for http authentication.
+ </description>
+ </property>
+ <property>
<name>ozone.recon.container.db.cache.size.mb</name>
<value>128</value>
<tag>RECON, PERFORMANCE</tag>
@@ -2363,108 +2407,13 @@
</property>
<property>
<name>hdds.tracing.enabled</name>
- <value>true</value>
+ <value>false</value>
<tag>OZONE, HDDS</tag>
<description>
If enabled, tracing information is sent to tracing server.
</description>
</property>
<property>
- <name>ozone.recon.sql.db.driver</name>
- <value>org.sqlite.JDBC</value>
- <tag>OZONE, RECON</tag>
- <description>
- Database driver class name available on the
- Ozone Recon classpath.
- </description>
- </property>
- <property>
- <name>ozone.recon.sql.db.jdbc.url</name>
- <value>jdbc:sqlite:${ozone.recon.db.dir}/ozone_recon_sqlite.db</value>
- <tag>OZONE, RECON</tag>
- <description>
- Ozone Recon SQL database jdbc url.
- </description>
- </property>
- <property>
- <name>ozone.recon.sql.db.username</name>
- <value/>
- <tag>OZONE, RECON</tag>
- <description>
- Ozone Recon SQL database username.
- </description>
- </property>
- <property>
- <name>ozone.recon.sql.db.password</name>
- <value/>
- <tag>OZONE, RECON</tag>
- <description>
- Ozone Recon database password.
- </description>
- </property>
- <property>
- <name>ozone.recon.sql.db.auto.commit</name>
- <value>false</value>
- <tag>OZONE, RECON</tag>
- <description>
- Sets the Ozone Recon database connection property of auto-commit to
- true/false.
- </description>
- </property>
- <property>
- <name>ozone.recon.sql.db.conn.timeout</name>
- <value>30000</value>
- <tag>OZONE, RECON</tag>
- <description>
- Sets time in milliseconds before call to getConnection is timed out.
- </description>
- </property>
- <property>
- <name>ozone.recon.sql.db.conn.max.active</name>
- <value>1</value>
- <tag>OZONE, RECON</tag>
- <description>
- The max active connections to the SQL database. The default SQLite
- database only allows single active connection, set this to a
- reasonable value like 10, for external production database.
- </description>
- </property>
- <property>
- <name>ozone.recon.sql.db.conn.max.age</name>
- <value>1800</value>
- <tag>OZONE, RECON</tag>
- <description>
- Sets maximum time a connection can be active in seconds.
- </description>
- </property>
- <property>
- <name>ozone.recon.sql.db.conn.idle.max.age</name>
- <value>3600</value>
- <tag>OZONE, RECON</tag>
- <description>
- Sets maximum time to live for idle connection in seconds.
- </description>
- </property>
- <property>
- <name>ozone.recon.sql.db.conn.idle.test.period</name>
- <value>60</value>
- <tag>OZONE, RECON</tag>
- <description>
- This sets the time (in seconds), for a connection to remain idle before
- sending a test query to the DB. This is useful to prevent a DB from
- timing out connections on its end.
- </description>
- </property>
- <property>
- <name>ozone.recon.sql.db.conn.idle.test</name>
- <value>SELECT 1</value>
- <tag>OZONE, RECON</tag>
- <description>
- The query to send to the DB to maintain keep-alives and test for dead
- connections.
- </description>
- </property>
- <property>
<name>ozone.recon.task.thread.count</name>
<value>1</value>
<tag>OZONE, RECON</tag>
@@ -2571,4 +2520,12 @@
Truststore password for HTTPS SSL configuration
</description>
</property>
+ <property>
+ <name>hdds.datanode.ratis.server.request.timeout</name>
+ <tag>OZONE, DATANODE</tag>
+ <value>2m</value>
+ <description>
+ Timeout for the request submitted directly to Ratis in datanode.
+ </description>
+ </property>
</configuration>
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHddsUtils.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHddsUtils.java
index ddb1f2b..7530bd0 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHddsUtils.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHddsUtils.java
@@ -25,18 +25,17 @@
import java.util.Map;
import java.util.Optional;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.test.LambdaTestUtils;
-import org.junit.Assert;
-import org.junit.Test;
import static org.apache.hadoop.hdds.HddsUtils.getSCMAddresses;
import static org.hamcrest.core.Is.is;
+import org.junit.Assert;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import org.junit.Test;
/**
* Testing HddsUtils.
@@ -76,7 +75,7 @@
@Test
public void testGetSCMAddresses() {
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
Collection<InetSocketAddress> addresses;
InetSocketAddress addr;
Iterator<InetSocketAddress> it;
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java
index ee724e2..5ab16ab 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java
@@ -17,26 +17,26 @@
*/
package org.apache.hadoop.hdds.conf;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.junit.Rule;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.Assert;
-import org.junit.rules.TemporaryFolder;
-
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.fs.Path;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
/**
* Test class for OzoneConfiguration.
*/
public class TestOzoneConfiguration {
- private Configuration conf;
+ private OzoneConfiguration conf;
@Rule
public TemporaryFolder tempConfigs = new TemporaryFolder();
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/TestDUFactory.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/TestDUFactory.java
index e651174..7a8701e 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/TestDUFactory.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/TestDUFactory.java
@@ -17,14 +17,14 @@
*/
package org.apache.hadoop.hdds.fs;
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Test;
-
import java.io.File;
import java.time.Duration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+
import static org.apache.hadoop.hdds.fs.DUFactory.Conf.configKeyForRefreshPeriod;
import static org.apache.hadoop.test.GenericTestUtils.getTestDir;
+import org.junit.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
@@ -40,7 +40,7 @@
@Test
public void testParams() {
- Configuration conf = new Configuration();
+ OzoneConfiguration conf = new OzoneConfiguration();
conf.set(configKeyForRefreshPeriod(), "1h");
File dir = getTestDir(getClass().getSimpleName());
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/TestDedicatedDiskSpaceUsageFactory.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/TestDedicatedDiskSpaceUsageFactory.java
index e3015b5..d0dfe60 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/TestDedicatedDiskSpaceUsageFactory.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/TestDedicatedDiskSpaceUsageFactory.java
@@ -17,14 +17,14 @@
*/
package org.apache.hadoop.hdds.fs;
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Test;
-
import java.io.File;
import java.time.Duration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+
import static org.apache.hadoop.hdds.fs.DedicatedDiskSpaceUsageFactory.Conf.configKeyForRefreshPeriod;
import static org.apache.hadoop.test.GenericTestUtils.getTestDir;
+import org.junit.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
@@ -41,7 +41,7 @@
@Test
public void testParams() {
- Configuration conf = new Configuration();
+ OzoneConfiguration conf = new OzoneConfiguration();
conf.set(configKeyForRefreshPeriod(), "2m");
File dir = getTestDir(getClass().getSimpleName());
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/TestSpaceUsageFactory.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/TestSpaceUsageFactory.java
index 09b8cc2..4f53b26 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/TestSpaceUsageFactory.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/fs/TestSpaceUsageFactory.java
@@ -17,19 +17,20 @@
*/
package org.apache.hadoop.hdds.fs;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.LoggerFactory;
-
import java.io.File;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
+
import static org.apache.hadoop.hdds.fs.SpaceUsageCheckFactory.Conf.configKeyForClassName;
+import org.junit.Before;
+import org.junit.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import org.slf4j.LoggerFactory;
/**
* Tests for {@link SpaceUsageCheckFactory}.
@@ -39,7 +40,8 @@
private LogCapturer capturer;
/**
- * Verifies that {@link SpaceUsageCheckFactory#create(Configuration)} creates
+ * Verifies that {@link SpaceUsageCheckFactory#create(ConfigurationSource)}
+ * creates
* the correct implementation if configured. This should be called from each
* specific implementation's test class.
* @return the instance created, so that further checks can done, if needed
@@ -47,7 +49,7 @@
protected static <T extends SpaceUsageCheckFactory> T testCreateViaConfig(
Class<T> factoryClass) {
- Configuration conf = configFor(factoryClass);
+ OzoneConfiguration conf = configFor(factoryClass);
SpaceUsageCheckFactory factory = SpaceUsageCheckFactory.create(conf);
@@ -104,10 +106,10 @@
"in log output, but only got: " + output);
}
- private static <T extends SpaceUsageCheckFactory> Configuration configFor(
- Class<T> factoryClass) {
+ private static <T extends SpaceUsageCheckFactory> OzoneConfiguration
+ configFor(Class<T> factoryClass) {
- Configuration conf = new Configuration();
+ OzoneConfiguration conf = new OzoneConfiguration();
conf.setClass(configKeyForClassName(),
factoryClass, SpaceUsageCheckFactory.class);
@@ -116,12 +118,12 @@
private static void testDefaultFactoryForBrokenImplementation(
Class<? extends SpaceUsageCheckFactory> brokenImplementationClass) {
- Configuration conf = configFor(brokenImplementationClass);
+ OzoneConfiguration conf = configFor(brokenImplementationClass);
assertCreatesDefaultImplementation(conf);
}
private void testDefaultFactoryForWrongConfig(String value) {
- Configuration conf = new Configuration();
+ OzoneConfiguration conf = new OzoneConfiguration();
conf.set(configKeyForClassName(), value);
assertCreatesDefaultImplementation(conf);
@@ -133,7 +135,8 @@
}
}
- private static void assertCreatesDefaultImplementation(Configuration conf) {
+ private static void assertCreatesDefaultImplementation(
+ OzoneConfiguration conf) {
// given
// conf
@@ -171,15 +174,16 @@
}
/**
- * Spy factory to verify {@link SpaceUsageCheckFactory#create(Configuration)}
+ * Spy factory to verify
+ * {@link SpaceUsageCheckFactory#create(ConfigurationSource)}
* properly configures it.
*/
public static final class SpyFactory implements SpaceUsageCheckFactory {
- private Configuration conf;
+ private ConfigurationSource conf;
@Override
- public SpaceUsageCheckFactory setConfiguration(Configuration config) {
+ public SpaceUsageCheckFactory setConfiguration(ConfigurationSource config) {
this.conf = config;
return this;
}
@@ -189,7 +193,7 @@
throw new UnsupportedOperationException();
}
- public Configuration getConf() {
+ public ConfigurationSource getConf() {
return conf;
}
}
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/net/TestNetworkTopologyImpl.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/net/TestNetworkTopologyImpl.java
index 6044666..c8dfd2c 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/net/TestNetworkTopologyImpl.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/net/TestNetworkTopologyImpl.java
@@ -17,23 +17,6 @@
*/
package org.apache.hadoop.hdds.scm.net;
-import org.apache.hadoop.conf.Configuration;
-import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT;
-import static org.apache.hadoop.hdds.scm.net.NetConstants.PATH_SEPARATOR_STR;
-import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT_SCHEMA;
-import static org.apache.hadoop.hdds.scm.net.NetConstants.REGION_SCHEMA;
-import static org.apache.hadoop.hdds.scm.net.NetConstants.DATACENTER_SCHEMA;
-import static org.apache.hadoop.hdds.scm.net.NetConstants.RACK_SCHEMA;
-import static org.apache.hadoop.hdds.scm.net.NetConstants.NODEGROUP_SCHEMA;
-import static org.apache.hadoop.hdds.scm.net.NetConstants.LEAF_SCHEMA;
-
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -42,6 +25,18 @@
import java.util.Map;
import java.util.Random;
import java.util.stream.Collectors;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+
+import static org.apache.hadoop.hdds.scm.net.NetConstants.DATACENTER_SCHEMA;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.LEAF_SCHEMA;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.NODEGROUP_SCHEMA;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.PATH_SEPARATOR_STR;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.RACK_SCHEMA;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.REGION_SCHEMA;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT_SCHEMA;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -49,9 +44,14 @@
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.junit.Assume.assumeTrue;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
-import org.junit.runner.RunWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/** Test the network topology functions. */
@RunWith(Parameterized.class)
@@ -221,7 +221,7 @@
@Test
public void testInitWithConfigFile() {
ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
- Configuration conf = new Configuration();
+ OzoneConfiguration conf = new OzoneConfiguration();
try {
String filePath = classLoader.getResource(
"./networkTopologyTestFiles/good.xml").getPath();
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/net/TestNodeSchemaManager.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/net/TestNodeSchemaManager.java
index 6698043..ae97155 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/net/TestNodeSchemaManager.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/net/TestNodeSchemaManager.java
@@ -17,19 +17,19 @@
*/
package org.apache.hadoop.hdds.scm.net;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import static org.apache.hadoop.hdds.scm.net.NetConstants.DEFAULT_NODEGROUP;
import static org.apache.hadoop.hdds.scm.net.NetConstants.DEFAULT_RACK;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/** Test the node schema loader. */
public class TestNodeSchemaManager {
@@ -38,10 +38,10 @@
private ClassLoader classLoader =
Thread.currentThread().getContextClassLoader();
private NodeSchemaManager manager;
- private Configuration conf;
+ private OzoneConfiguration conf;
public TestNodeSchemaManager() {
- conf = new Configuration();
+ conf = new OzoneConfiguration();
String filePath = classLoader.getResource(
"./networkTopologyTestFiles/good.xml").getPath();
conf.set(ScmConfigKeys.OZONE_SCM_NETWORK_TOPOLOGY_SCHEMA_FILE, filePath);
diff --git a/hadoop-hdds/config/pom.xml b/hadoop-hdds/config/pom.xml
index f8aac02..105e8ac 100644
--- a/hadoop-hdds/config/pom.xml
+++ b/hadoop-hdds/config/pom.xml
@@ -33,6 +33,15 @@
</properties>
<dependencies>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-test-utils</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>junit</groupId>
diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationReflectionUtil.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationReflectionUtil.java
new file mode 100644
index 0000000..48f06ea
--- /dev/null
+++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationReflectionUtil.java
@@ -0,0 +1,159 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.conf;
+
+import javax.annotation.PostConstruct;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+
+/**
+ * Reflection utilities for configuration injection.
+ */
+public final class ConfigurationReflectionUtil {
+
+ private ConfigurationReflectionUtil() {
+ }
+
+ public static <T> void injectConfiguration(
+ ConfigurationSource configuration,
+ Class<T> configurationClass,
+ T configObject, String prefix) {
+ injectConfigurationToObject(configuration, configurationClass, configObject,
+ prefix);
+ Class<? super T> superClass = configurationClass.getSuperclass();
+ while (superClass != null) {
+ injectConfigurationToObject(configuration, superClass, configObject,
+ prefix);
+ superClass = superClass.getSuperclass();
+ }
+ }
+
+ public static <T> void injectConfigurationToObject(ConfigurationSource from,
+ Class<T> configurationClass,
+ T configuration,
+ String prefix) {
+ for (Field field : configurationClass.getDeclaredFields()) {
+ if (field.isAnnotationPresent(Config.class)) {
+
+ String fieldLocation =
+ configurationClass + "." + field.getName();
+
+ Config configAnnotation = field.getAnnotation(Config.class);
+
+ String key = prefix + "." + configAnnotation.key();
+
+ ConfigType type = configAnnotation.type();
+
+ if (type == ConfigType.AUTO) {
+ type = detectConfigType(field.getType(), fieldLocation);
+ }
+
+ //Note: default value is handled by ozone-default.xml. Here we can
+ //use any default.
+ try {
+ switch (type) {
+ case STRING:
+ forcedFieldSet(field, configuration, from.get(key));
+ break;
+ case INT:
+ forcedFieldSet(field, configuration, from.getInt(key, 0));
+ break;
+ case BOOLEAN:
+ forcedFieldSet(field, configuration, from.getBoolean(key, false));
+ break;
+ case LONG:
+ forcedFieldSet(field, configuration, from.getLong(key, 0));
+ break;
+ case TIME:
+ forcedFieldSet(field, configuration,
+ from.getTimeDuration(key, "0s", configAnnotation.timeUnit()));
+ break;
+ default:
+ throw new ConfigurationException(
+ "Unsupported ConfigType " + type + " on " + fieldLocation);
+ }
+ } catch (IllegalAccessException e) {
+ throw new ConfigurationException(
+ "Can't inject configuration to " + fieldLocation, e);
+ }
+
+ }
+ }
+ }
+
+ /**
+ * Set the value of one field even if it's private.
+ */
+ private static <T> void forcedFieldSet(Field field, T object, Object value)
+ throws IllegalAccessException {
+ boolean accessChanged = false;
+ if (!field.isAccessible()) {
+ field.setAccessible(true);
+ accessChanged = true;
+ }
+ field.set(object, value);
+ if (accessChanged) {
+ field.setAccessible(false);
+ }
+ }
+
+ private static ConfigType detectConfigType(Class<?> parameterType,
+ String methodLocation) {
+ ConfigType type;
+ if (parameterType == String.class) {
+ type = ConfigType.STRING;
+ } else if (parameterType == Integer.class || parameterType == int.class) {
+ type = ConfigType.INT;
+ } else if (parameterType == Long.class || parameterType == long.class) {
+ type = ConfigType.LONG;
+ } else if (parameterType == Boolean.class
+ || parameterType == boolean.class) {
+ type = ConfigType.BOOLEAN;
+ } else {
+ throw new ConfigurationException(
+ "Unsupported configuration type " + parameterType + " in "
+ + methodLocation);
+ }
+ return type;
+ }
+
+ public static <T> void callPostConstruct(Class<T> configurationClass,
+ T configObject) {
+ for (Method method : configurationClass.getMethods()) {
+ if (method.isAnnotationPresent(PostConstruct.class)) {
+ try {
+ method.invoke(configObject);
+ } catch (IllegalAccessException ex) {
+ throw new IllegalArgumentException(
+ "@PostConstruct method in " + configurationClass
+ + " is not accessible");
+ } catch (InvocationTargetException e) {
+ if (e.getCause() instanceof RuntimeException) {
+ throw (RuntimeException) e.getCause();
+ } else {
+ throw new IllegalArgumentException(
+ "@PostConstruct can't be executed on " + configurationClass
+ + " after configObject "
+ + "injection", e);
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationSource.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationSource.java
new file mode 100644
index 0000000..85d2b0b
--- /dev/null
+++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationSource.java
@@ -0,0 +1,289 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.conf;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Lightweight interface to defined the contract of the Configuration objects.
+ */
+public interface ConfigurationSource {
+
+ String[] EMPTY_STRING_ARRAY = {};
+
+ String get(String key);
+
+ Collection<String> getConfigKeys();
+
+ char[] getPassword(String key) throws IOException;
+
+ @Deprecated
+ //TODO: user read only configs and don't use it to store actual port
+ // numbers.
+ void set(String key, String value);
+
+ default String get(String key, String defaultValue) {
+ String value = get(key);
+ return value != null ? value : defaultValue;
+ }
+
+ default int getInt(String key, int defaultValue) {
+ String value = get(key);
+ return value != null ? Integer.parseInt(value) : defaultValue;
+ }
+
+ /**
+ * Get the value of the <code>name</code> property as a set of comma-delimited
+ * <code>int</code> values.
+ * <p>
+ * If no such property exists, an empty array is returned.
+ *
+ * @param name property name
+ * @return property value interpreted as an array of comma-delimited
+ * <code>int</code> values
+ */
+ default int[] getInts(String name) {
+ String[] strings = getTrimmedStrings(name);
+ int[] ints = new int[strings.length];
+ for (int i = 0; i < strings.length; i++) {
+ ints[i] = Integer.parseInt(strings[i]);
+ }
+ return ints;
+ }
+
+ default long getLong(String key, long defaultValue) {
+ String value = get(key);
+ return value != null ? Long.parseLong(value) : defaultValue;
+ }
+
+ default boolean getBoolean(String key, boolean defaultValue) {
+ String value = get(key);
+ return value != null ? Boolean.parseBoolean(value) : defaultValue;
+ }
+
+ default float getFloat(String key, float defaultValue) {
+ String value = get(key);
+ return value != null ? Float.parseFloat(value) : defaultValue;
+ }
+
+ default double getDouble(String key, double defaultValue) {
+ String value = get(key);
+ return value != null ? Double.parseDouble(value) : defaultValue;
+ }
+
+ default String getTrimmed(String key) {
+ String value = get(key);
+ return value != null ? value.trim() : null;
+ }
+
+ default String getTrimmed(String key, String defaultValue) {
+ String value = getTrimmed(key);
+ return value != null ? value : defaultValue;
+ }
+
+ default String[] getTrimmedStrings(String name) {
+ String valueString = get(name);
+ if (null == valueString || valueString.trim().isEmpty()) {
+ return EMPTY_STRING_ARRAY;
+ }
+
+ return valueString.trim().split("\\s*[,\n]\\s*");
+ }
+
+ default Map<String, String> getPropsWithPrefix(String confPrefix) {
+ Map<String, String> configMap = new HashMap<>();
+ for (String name : getConfigKeys()) {
+ if (name.startsWith(confPrefix)) {
+ String value = get(name);
+ String keyName = name.substring(confPrefix.length());
+ configMap.put(keyName, value);
+ }
+ }
+ return configMap;
+ }
+
+ /**
+ * Create a Configuration object and inject the required configuration values.
+ *
+ * @param configurationClass The class where the fields are annotated with
+ * the configuration.
+ * @return Initiated java object where the config fields are injected.
+ */
+ default <T> T getObject(Class<T> configurationClass) {
+
+ T configObject;
+
+ try {
+ configObject = configurationClass.newInstance();
+ } catch (InstantiationException | IllegalAccessException e) {
+ throw new ConfigurationException(
+ "Configuration class can't be created: " + configurationClass, e);
+ }
+ ConfigGroup configGroup =
+ configurationClass.getAnnotation(ConfigGroup.class);
+
+ String prefix = configGroup.prefix();
+
+ ConfigurationReflectionUtil
+ .injectConfiguration(this, configurationClass, configObject,
+ prefix);
+
+ ConfigurationReflectionUtil
+ .callPostConstruct(configurationClass, configObject);
+
+ return configObject;
+
+ }
+
+ /**
+ * Get the value of the <code>name</code> property as a <code>Class</code>
+ * implementing the interface specified by <code>xface</code>.
+ * <p>
+ * If no such property is specified, then <code>defaultValue</code> is
+ * returned.
+ * <p>
+ * An exception is thrown if the returned class does not implement the named
+ * interface.
+ *
+ * @param name the class name.
+ * @param defaultValue default value.
+ * @param xface the interface implemented by the named class.
+ * @return property value as a <code>Class</code>,
+ * or <code>defaultValue</code>.
+ */
+ default <U> Class<? extends U> getClass(String name,
+ Class<? extends U> defaultValue,
+ Class<U> xface) {
+ try {
+ Class<?> theClass = getClass(name, defaultValue);
+ if (theClass != null && !xface.isAssignableFrom(theClass)) {
+ throw new RuntimeException(theClass + " not " + xface.getName());
+ } else if (theClass != null) {
+ return theClass.asSubclass(xface);
+ } else {
+ return null;
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Get the value of the <code>name</code> property as a <code>Class</code>.
+ * If no such property is specified, then <code>defaultValue</code> is
+ * returned.
+ *
+ * @param name the class name.
+ * @param defaultValue default value.
+ * @return property value as a <code>Class</code>,
+ * or <code>defaultValue</code>.
+ */
+ default Class<?> getClass(String name, Class<?> defaultValue) {
+ String valueString = getTrimmed(name);
+ if (valueString == null) {
+ return defaultValue;
+ }
+ try {
+ return Class.forName(name);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ default Class<?>[] getClasses(String name, Class<?>... defaultValue) {
+ String valueString = get(name);
+ if (null == valueString) {
+ return defaultValue;
+ }
+ String[] classnames = getTrimmedStrings(name);
+ try {
+ Class<?>[] classes = new Class<?>[classnames.length];
+ for (int i = 0; i < classnames.length; i++) {
+ classes[i] = Class.forName(classnames[i]);
+ }
+ return classes;
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ default long getTimeDuration(String name, long defaultValue,
+ TimeUnit unit) {
+ String vStr = get(name);
+ if (null == vStr) {
+ return defaultValue;
+ } else {
+ return TimeDurationUtil.getTimeDurationHelper(name, vStr, unit);
+ }
+ }
+
+ default long getTimeDuration(String name, String defaultValue,
+ TimeUnit unit) {
+ String vStr = get(name);
+ if (null == vStr) {
+ return TimeDurationUtil.getTimeDurationHelper(name, defaultValue, unit);
+ } else {
+ return TimeDurationUtil.getTimeDurationHelper(name, vStr, unit);
+ }
+ }
+
+ default double getStorageSize(String name, String defaultValue,
+ StorageUnit targetUnit) {
+ String vString = get(name);
+ if (vString == null) {
+ vString = defaultValue;
+ }
+
+ // Please note: There is a bit of subtlety here. If the user specifies
+ // the default unit as "1GB", but the requested unit is MB, we will return
+ // the format in MB even thought the default string is specified in GB.
+
+ // Converts a string like "1GB" to to unit specified in targetUnit.
+
+ StorageSize measure = StorageSize.parse(vString);
+
+ double byteValue = measure.getUnit().toBytes(measure.getValue());
+ return targetUnit.fromBytes(byteValue);
+ }
+
+ default Collection<String> getTrimmedStringCollection(String key) {
+ return Arrays.asList(getTrimmedStrings(key));
+ }
+
+ /**
+ * Return value matching this enumerated type.
+ * Note that the returned value is trimmed by this method.
+ *
+ * @param name Property name
+ * @param defaultValue Value returned if no mapping exists
+ * @throws IllegalArgumentException If mapping is illegal for the type
+ * provided
+ */
+ default <T extends Enum<T>> T getEnum(String name, T defaultValue) {
+ final String val = getTrimmed(name);
+ return null == val
+ ? defaultValue
+ : Enum.valueOf(defaultValue.getDeclaringClass(), val);
+ }
+
+}
diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/StorageSize.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/StorageSize.java
new file mode 100644
index 0000000..15016be
--- /dev/null
+++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/StorageSize.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.conf;
+
+import java.util.Locale;
+
+/**
+ * A class that contains the numeric value and the unit of measure.
+ */
+public class StorageSize {
+ private final StorageUnit unit;
+ private final double value;
+
+ /**
+ * Constucts a Storage Measure, which contains the value and the unit of
+ * measure.
+ *
+ * @param unit - Unit of Measure
+ * @param value - Numeric value.
+ */
+ public StorageSize(StorageUnit unit, double value) {
+ this.unit = unit;
+ this.value = value;
+ }
+
+ private static void checkState(boolean state, String errorString) {
+ if (!state) {
+ throw new IllegalStateException(errorString);
+ }
+ }
+
+ public static StorageSize parse(String value) {
+ checkState(value != null && value.length() > 0, "value cannot be blank");
+ String sanitizedValue = value.trim().toLowerCase(Locale.ENGLISH);
+ StorageUnit parsedUnit = null;
+ for (StorageUnit unit : StorageUnit.values()) {
+ if (sanitizedValue.endsWith(unit.getShortName()) ||
+ sanitizedValue.endsWith(unit.getLongName()) ||
+ sanitizedValue.endsWith(unit.getSuffixChar())) {
+ parsedUnit = unit;
+ break;
+ }
+ }
+
+ if (parsedUnit == null) {
+ throw new IllegalArgumentException(value + " is not in expected format." +
+ "Expected format is <number><unit>. e.g. 1000MB");
+ }
+
+ String suffix = "";
+ boolean found = false;
+
+ // We are trying to get the longest match first, so the order of
+ // matching is getLongName, getShortName and then getSuffixChar.
+ if (!found && sanitizedValue.endsWith(parsedUnit.getLongName())) {
+ found = true;
+ suffix = parsedUnit.getLongName();
+ }
+
+ if (!found && sanitizedValue.endsWith(parsedUnit.getShortName())) {
+ found = true;
+ suffix = parsedUnit.getShortName();
+ }
+
+ if (!found && sanitizedValue.endsWith(parsedUnit.getSuffixChar())) {
+ found = true;
+ suffix = parsedUnit.getSuffixChar();
+ }
+
+ checkState(found, "Something is wrong, we have to find a " +
+ "match. Internal error.");
+
+ String valString =
+ sanitizedValue.substring(0, value.length() - suffix.length());
+ return new StorageSize(parsedUnit, Double.parseDouble(valString));
+
+ }
+
+ public StorageUnit getUnit() {
+ return unit;
+ }
+
+ public double getValue() {
+ return value;
+ }
+
+}
diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/StorageUnit.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/StorageUnit.java
new file mode 100644
index 0000000..6678aa4
--- /dev/null
+++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/StorageUnit.java
@@ -0,0 +1,529 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.conf;
+
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+
+/**
+ * Enum to represent storage unit.
+ */
+public enum StorageUnit {
+ /*
+ We rely on BYTES being the last to get longest matching short names first.
+ The short name of bytes is b and it will match with other longer names.
+
+ if we change this order, the corresponding code in
+ Configuration#parseStorageUnit needs to be changed too, since values()
+ call returns the Enums in declared order and we depend on it.
+ */
+
+ EB {
+ @Override
+ public double toBytes(double value) {
+ return multiply(value, EXABYTES);
+ }
+
+ @Override
+ public double toKBs(double value) {
+ return multiply(value, EXABYTES / KILOBYTES);
+ }
+
+ @Override
+ public double toMBs(double value) {
+ return multiply(value, EXABYTES / MEGABYTES);
+ }
+
+ @Override
+ public double toGBs(double value) {
+ return multiply(value, EXABYTES / GIGABYTES);
+ }
+
+ @Override
+ public double toTBs(double value) {
+ return multiply(value, EXABYTES / TERABYTES);
+ }
+
+ @Override
+ public double toPBs(double value) {
+ return multiply(value, EXABYTES / PETABYTES);
+ }
+
+ @Override
+ public double toEBs(double value) {
+ return value;
+ }
+
+ @Override
+ public String getLongName() {
+ return "exabytes";
+ }
+
+ @Override
+ public String getShortName() {
+ return "eb";
+ }
+
+ @Override
+ public String getSuffixChar() {
+ return "e";
+ }
+
+ @Override
+ public double getDefault(double value) {
+ return toEBs(value);
+ }
+
+ @Override
+ public double fromBytes(double value) {
+ return divide(value, EXABYTES);
+ }
+ },
+ PB {
+ @Override
+ public double toBytes(double value) {
+ return multiply(value, PETABYTES);
+ }
+
+ @Override
+ public double toKBs(double value) {
+ return multiply(value, PETABYTES / KILOBYTES);
+ }
+
+ @Override
+ public double toMBs(double value) {
+ return multiply(value, PETABYTES / MEGABYTES);
+ }
+
+ @Override
+ public double toGBs(double value) {
+ return multiply(value, PETABYTES / GIGABYTES);
+ }
+
+ @Override
+ public double toTBs(double value) {
+ return multiply(value, PETABYTES / TERABYTES);
+ }
+
+ @Override
+ public double toPBs(double value) {
+ return value;
+ }
+
+ @Override
+ public double toEBs(double value) {
+ return divide(value, EXABYTES / PETABYTES);
+ }
+
+ @Override
+ public String getLongName() {
+ return "petabytes";
+ }
+
+ @Override
+ public String getShortName() {
+ return "pb";
+ }
+
+ @Override
+ public String getSuffixChar() {
+ return "p";
+ }
+
+ @Override
+ public double getDefault(double value) {
+ return toPBs(value);
+ }
+
+ @Override
+ public double fromBytes(double value) {
+ return divide(value, PETABYTES);
+ }
+ },
+ TB {
+ @Override
+ public double toBytes(double value) {
+ return multiply(value, TERABYTES);
+ }
+
+ @Override
+ public double toKBs(double value) {
+ return multiply(value, TERABYTES / KILOBYTES);
+ }
+
+ @Override
+ public double toMBs(double value) {
+ return multiply(value, TERABYTES / MEGABYTES);
+ }
+
+ @Override
+ public double toGBs(double value) {
+ return multiply(value, TERABYTES / GIGABYTES);
+ }
+
+ @Override
+ public double toTBs(double value) {
+ return value;
+ }
+
+ @Override
+ public double toPBs(double value) {
+ return divide(value, PETABYTES / TERABYTES);
+ }
+
+ @Override
+ public double toEBs(double value) {
+ return divide(value, EXABYTES / TERABYTES);
+ }
+
+ @Override
+ public String getLongName() {
+ return "terabytes";
+ }
+
+ @Override
+ public String getShortName() {
+ return "tb";
+ }
+
+ @Override
+ public String getSuffixChar() {
+ return "t";
+ }
+
+ @Override
+ public double getDefault(double value) {
+ return toTBs(value);
+ }
+
+ @Override
+ public double fromBytes(double value) {
+ return divide(value, TERABYTES);
+ }
+ },
+ GB {
+ @Override
+ public double toBytes(double value) {
+ return multiply(value, GIGABYTES);
+ }
+
+ @Override
+ public double toKBs(double value) {
+ return multiply(value, GIGABYTES / KILOBYTES);
+ }
+
+ @Override
+ public double toMBs(double value) {
+ return multiply(value, GIGABYTES / MEGABYTES);
+ }
+
+ @Override
+ public double toGBs(double value) {
+ return value;
+ }
+
+ @Override
+ public double toTBs(double value) {
+ return divide(value, TERABYTES / GIGABYTES);
+ }
+
+ @Override
+ public double toPBs(double value) {
+ return divide(value, PETABYTES / GIGABYTES);
+ }
+
+ @Override
+ public double toEBs(double value) {
+ return divide(value, EXABYTES / GIGABYTES);
+ }
+
+ @Override
+ public String getLongName() {
+ return "gigabytes";
+ }
+
+ @Override
+ public String getShortName() {
+ return "gb";
+ }
+
+ @Override
+ public String getSuffixChar() {
+ return "g";
+ }
+
+ @Override
+ public double getDefault(double value) {
+ return toGBs(value);
+ }
+
+ @Override
+ public double fromBytes(double value) {
+ return divide(value, GIGABYTES);
+ }
+ },
+ MB {
+ @Override
+ public double toBytes(double value) {
+ return multiply(value, MEGABYTES);
+ }
+
+ @Override
+ public double toKBs(double value) {
+ return multiply(value, MEGABYTES / KILOBYTES);
+ }
+
+ @Override
+ public double toMBs(double value) {
+ return value;
+ }
+
+ @Override
+ public double toGBs(double value) {
+ return divide(value, GIGABYTES / MEGABYTES);
+ }
+
+ @Override
+ public double toTBs(double value) {
+ return divide(value, TERABYTES / MEGABYTES);
+ }
+
+ @Override
+ public double toPBs(double value) {
+ return divide(value, PETABYTES / MEGABYTES);
+ }
+
+ @Override
+ public double toEBs(double value) {
+ return divide(value, EXABYTES / MEGABYTES);
+ }
+
+ @Override
+ public String getLongName() {
+ return "megabytes";
+ }
+
+ @Override
+ public String getShortName() {
+ return "mb";
+ }
+
+ @Override
+ public String getSuffixChar() {
+ return "m";
+ }
+
+ @Override
+ public double fromBytes(double value) {
+ return divide(value, MEGABYTES);
+ }
+
+ @Override
+ public double getDefault(double value) {
+ return toMBs(value);
+ }
+ },
+ KB {
+ @Override
+ public double toBytes(double value) {
+ return multiply(value, KILOBYTES);
+ }
+
+ @Override
+ public double toKBs(double value) {
+ return value;
+ }
+
+ @Override
+ public double toMBs(double value) {
+ return divide(value, MEGABYTES / KILOBYTES);
+ }
+
+ @Override
+ public double toGBs(double value) {
+ return divide(value, GIGABYTES / KILOBYTES);
+ }
+
+ @Override
+ public double toTBs(double value) {
+ return divide(value, TERABYTES / KILOBYTES);
+ }
+
+ @Override
+ public double toPBs(double value) {
+ return divide(value, PETABYTES / KILOBYTES);
+ }
+
+ @Override
+ public double toEBs(double value) {
+ return divide(value, EXABYTES / KILOBYTES);
+ }
+
+ @Override
+ public String getLongName() {
+ return "kilobytes";
+ }
+
+ @Override
+ public String getShortName() {
+ return "kb";
+ }
+
+ @Override
+ public String getSuffixChar() {
+ return "k";
+ }
+
+ @Override
+ public double getDefault(double value) {
+ return toKBs(value);
+ }
+
+ @Override
+ public double fromBytes(double value) {
+ return divide(value, KILOBYTES);
+ }
+ },
+ BYTES {
+ @Override
+ public double toBytes(double value) {
+ return value;
+ }
+
+ @Override
+ public double toKBs(double value) {
+ return divide(value, KILOBYTES);
+ }
+
+ @Override
+ public double toMBs(double value) {
+ return divide(value, MEGABYTES);
+ }
+
+ @Override
+ public double toGBs(double value) {
+ return divide(value, GIGABYTES);
+ }
+
+ @Override
+ public double toTBs(double value) {
+ return divide(value, TERABYTES);
+ }
+
+ @Override
+ public double toPBs(double value) {
+ return divide(value, PETABYTES);
+ }
+
+ @Override
+ public double toEBs(double value) {
+ return divide(value, EXABYTES);
+ }
+
+ @Override
+ public String getLongName() {
+ return "bytes";
+ }
+
+ @Override
+ public String getShortName() {
+ return "b";
+ }
+
+ @Override
+ public String getSuffixChar() {
+ return "b";
+ }
+
+ @Override
+ public double getDefault(double value) {
+ return toBytes(value);
+ }
+
+ @Override
+ public double fromBytes(double value) {
+ return value;
+ }
+ };
+
+ private static final double BYTE = 1L;
+ private static final double KILOBYTES = BYTE * 1024L;
+ private static final double MEGABYTES = KILOBYTES * 1024L;
+ private static final double GIGABYTES = MEGABYTES * 1024L;
+ private static final double TERABYTES = GIGABYTES * 1024L;
+ private static final double PETABYTES = TERABYTES * 1024L;
+ private static final double EXABYTES = PETABYTES * 1024L;
+ private static final int PRECISION = 4;
+
+ /**
+ * Using BigDecimal to avoid issues with overflow and underflow.
+ *
+ * @param value - value
+ * @param divisor - divisor.
+ * @return -- returns a double that represents this value
+ */
+ private static double divide(double value, double divisor) {
+ BigDecimal val = new BigDecimal(value);
+ BigDecimal bDivisor = new BigDecimal(divisor);
+ return val.divide(bDivisor).setScale(PRECISION, RoundingMode.HALF_UP)
+ .doubleValue();
+ }
+
+ /**
+ * Using BigDecimal so we can throw if we are overflowing the Long.Max.
+ *
+ * @param first - First Num.
+ * @param second - Second Num.
+ * @return Returns a double
+ */
+ private static double multiply(double first, double second) {
+ BigDecimal firstVal = new BigDecimal(first);
+ BigDecimal secondVal = new BigDecimal(second);
+ return firstVal.multiply(secondVal)
+ .setScale(PRECISION, RoundingMode.HALF_UP).doubleValue();
+ }
+
+ public abstract double toBytes(double value);
+
+ public abstract double toKBs(double value);
+
+ public abstract double toMBs(double value);
+
+ public abstract double toGBs(double value);
+
+ public abstract double toTBs(double value);
+
+ public abstract double toPBs(double value);
+
+ public abstract double toEBs(double value);
+
+ public abstract String getLongName();
+
+ public abstract String getShortName();
+
+ public abstract String getSuffixChar();
+
+ public abstract double getDefault(double value);
+
+ public abstract double fromBytes(double value);
+
+ public String toString() {
+ return getLongName();
+ }
+
+}
diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/TimeDurationUtil.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/TimeDurationUtil.java
new file mode 100644
index 0000000..2bbdecf
--- /dev/null
+++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/TimeDurationUtil.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.conf;
+
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utility to handle time duration.
+ */
+public final class TimeDurationUtil {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(TimeDurationUtil.class);
+
+ private TimeDurationUtil() {
+ }
+
+ /**
+ * Return time duration in the given time unit. Valid units are encoded in
+ * properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
+ * (ms), seconds (s), minutes (m), hours (h), and days (d).
+ *
+ * @param name Property name
+ * @param vStr The string value with time unit suffix to be converted.
+ * @param unit Unit to convert the stored property, if it exists.
+ */
+ public static long getTimeDurationHelper(String name, String vStr,
+ TimeUnit unit) {
+ vStr = vStr.trim();
+ vStr = vStr.toLowerCase();
+ ParsedTimeDuration vUnit = ParsedTimeDuration.unitFor(vStr);
+ if (null == vUnit) {
+ LOG.warn("No unit for " + name + "(" + vStr + ") assuming " + unit);
+ vUnit = ParsedTimeDuration.unitFor(unit);
+ } else {
+ vStr = vStr.substring(0, vStr.lastIndexOf(vUnit.suffix()));
+ }
+
+ long raw = Long.parseLong(vStr);
+ long converted = unit.convert(raw, vUnit.unit());
+ if (vUnit.unit().convert(converted, unit) < raw) {
+ LOG.warn("Possible loss of precision converting " + vStr
+ + vUnit.suffix() + " to " + unit + " for " + name);
+ }
+ return converted;
+ }
+
+ enum ParsedTimeDuration {
+ NS {
+ TimeUnit unit() {
+ return TimeUnit.NANOSECONDS;
+ }
+
+ String suffix() {
+ return "ns";
+ }
+ },
+ US {
+ TimeUnit unit() {
+ return TimeUnit.MICROSECONDS;
+ }
+
+ String suffix() {
+ return "us";
+ }
+ },
+ MS {
+ TimeUnit unit() {
+ return TimeUnit.MILLISECONDS;
+ }
+
+ String suffix() {
+ return "ms";
+ }
+ },
+ S {
+ TimeUnit unit() {
+ return TimeUnit.SECONDS;
+ }
+
+ String suffix() {
+ return "s";
+ }
+ },
+ M {
+ TimeUnit unit() {
+ return TimeUnit.MINUTES;
+ }
+
+ String suffix() {
+ return "m";
+ }
+ },
+ H {
+ TimeUnit unit() {
+ return TimeUnit.HOURS;
+ }
+
+ String suffix() {
+ return "h";
+ }
+ },
+ D {
+ TimeUnit unit() {
+ return TimeUnit.DAYS;
+ }
+
+ String suffix() {
+ return "d";
+ }
+ };
+
+ abstract TimeUnit unit();
+
+ abstract String suffix();
+
+ static ParsedTimeDuration unitFor(String s) {
+ for (ParsedTimeDuration ptd : values()) {
+ // iteration order is in decl order, so SECONDS matched last
+ if (s.endsWith(ptd.suffix())) {
+ return ptd;
+ }
+ }
+ return null;
+ }
+
+ static ParsedTimeDuration unitFor(TimeUnit unit) {
+ for (ParsedTimeDuration ptd : values()) {
+ if (ptd.unit() == unit) {
+ return ptd;
+ }
+ }
+ return null;
+ }
+ }
+}
diff --git a/hadoop-hdds/container-service/pom.xml b/hadoop-hdds/container-service/pom.xml
index a785276..c948db5 100644
--- a/hadoop-hdds/container-service/pom.xml
+++ b/hadoop-hdds/container-service/pom.xml
@@ -35,6 +35,15 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-test-utils</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-common</artifactId>
<type>test-jar</type>
</dependency>
@@ -42,10 +51,10 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-server-framework</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdds-client</artifactId>
- </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-client</artifactId>
+ </dependency>
<dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
@@ -60,36 +69,30 @@
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
- <version>1.16</version>
</dependency>
<dependency>
<groupId>com.github.spotbugs</groupId>
<artifactId>spotbugs</artifactId>
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-module-junit4</artifactId>
- <version>2.0.4</version>
- <scope>test</scope>
- </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-api-mockito2</artifactId>
- <version>2.0.4</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
+ <artifactId>hadoop-hdds-hadoop-dependency-test</artifactId>
<scope>test</scope>
- <type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop.version}</version>
<scope>test</scope>
<type>test-jar</type>
</dependency>
@@ -98,29 +101,31 @@
<build>
<plugins>
<plugin>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-maven-plugins</artifactId>
+ <groupId>com.salesforce.servicelibs</groupId>
+ <artifactId>proto-backwards-compatibility</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.xolstice.maven.plugins</groupId>
+ <artifactId>protobuf-maven-plugin</artifactId>
+ <version>${protobuf-maven-plugin.version}</version>
+ <extensions>true</extensions>
<executions>
<execution>
<id>compile-protoc</id>
<goals>
- <goal>protoc</goal>
+ <goal>compile</goal>
+ <goal>test-compile</goal>
</goals>
<configuration>
- <protocVersion>${protobuf.version}</protocVersion>
- <protocCommand>${protoc.path}</protocCommand>
- <imports>
+ <additionalProtoPathElements>
<param>
${basedir}/../../hadoop-hdds/common/src/main/proto/
</param>
<param>${basedir}/src/main/proto</param>
- </imports>
- <source>
- <directory>${basedir}/src/main/proto</directory>
- <includes>
- <include>StorageContainerDatanodeProtocol.proto</include>
- </includes>
- </source>
+ </additionalProtoPathElements>
+ <protocArtifact>
+ com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}
+ </protocArtifact>
</configuration>
</execution>
</executions>
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeHttpServer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeHttpServer.java
index fe2d065..f533a26 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeHttpServer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeHttpServer.java
@@ -19,8 +19,8 @@
import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.server.http.BaseHttpServer;
/**
@@ -30,7 +30,7 @@
*/
public class HddsDatanodeHttpServer extends BaseHttpServer {
- public HddsDatanodeHttpServer(Configuration conf) throws IOException {
+ public HddsDatanodeHttpServer(OzoneConfiguration conf) throws IOException {
super(conf, "hddsDatanode");
}
@@ -83,4 +83,14 @@
protected String getEnabledKey() {
return HddsConfigKeys.HDDS_DATANODE_HTTP_ENABLED_KEY;
}
+
+ @Override
+ protected String getHttpAuthType() {
+ return HddsConfigKeys.HDDS_DATANODE_HTTP_AUTH_TYPE;
+ }
+
+ @Override
+ protected String getHttpAuthConfigPrefix() {
+ return HddsConfigKeys.OZONE_DATANODE_HTTP_AUTH_CONFIG_PREFIX;
+ }
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
index 6a6d718..e811a88 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
@@ -29,11 +29,11 @@
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.DFSConfigKeysLegacy;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.cli.GenericCli;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto;
@@ -357,7 +357,7 @@
* @param config
* */
@VisibleForTesting
- public PKCS10CertificationRequest getCSR(Configuration config)
+ public PKCS10CertificationRequest getCSR(ConfigurationSource config)
throws IOException {
CertificateSignRequest.Builder builder = dnCertClient.getCSRBuilder();
KeyPair keyPair = new KeyPair(dnCertClient.getPublicKey(),
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java
index 3de9579..e07f626 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.ozone.container.common.helpers;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.DFSConfigKeysLegacy;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.metrics2.MetricsSystem;
@@ -86,7 +86,7 @@
}
}
- public static ContainerMetrics create(Configuration conf) {
+ public static ContainerMetrics create(ConfigurationSource conf) {
MetricsSystem ms = DefaultMetricsSystem.instance();
// Percentile measurement is off by default, by watching no intervals
int[] intervals =
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkLayOutVersion.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkLayOutVersion.java
index a5bcc22..ff4ba41 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkLayOutVersion.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkLayOutVersion.java
@@ -18,20 +18,20 @@
package org.apache.hadoop.ozone.container.common.impl;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.client.BlockID;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.File;
import java.util.List;
+import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.UNABLE_TO_FIND_DATA_DIR;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Defines layout versions for the Chunks.
@@ -40,17 +40,15 @@
FILE_PER_CHUNK(1, "One file per chunk") {
@Override
- public File getChunkFile(ContainerData containerData, BlockID blockID,
+ public File getChunkFile(File chunkDir, BlockID blockID,
ChunkInfo info) throws StorageContainerException {
- File chunksLoc = verifyChunkDirExists(containerData);
- return chunksLoc.toPath().resolve(info.getChunkName()).toFile();
+ return chunkDir.toPath().resolve(info.getChunkName()).toFile();
}
},
FILE_PER_BLOCK(2, "One file per block") {
@Override
- public File getChunkFile(ContainerData containerData, BlockID blockID,
+ public File getChunkFile(File chunkDir, BlockID blockID,
ChunkInfo info) throws StorageContainerException {
- File chunkDir = verifyChunkDirExists(containerData);
return new File(chunkDir, blockID.getLocalID() + ".block");
}
};
@@ -94,7 +92,8 @@
/**
* @return the latest version.
*/
- public static ChunkLayOutVersion getConfiguredVersion(Configuration conf) {
+ public static ChunkLayOutVersion getConfiguredVersion(
+ ConfigurationSource conf) {
try {
return conf.getEnum(ScmConfigKeys.OZONE_SCM_CHUNK_LAYOUT_KEY,
DEFAULT_LAYOUT);
@@ -117,9 +116,15 @@
return description;
}
- public abstract File getChunkFile(ContainerData containerData,
+ public abstract File getChunkFile(File chunkDir,
BlockID blockID, ChunkInfo info) throws StorageContainerException;
+ public File getChunkFile(ContainerData containerData, BlockID blockID,
+ ChunkInfo info) throws StorageContainerException {
+ File chunksLoc = verifyChunkDirExists(containerData);
+ return getChunkFile(chunksLoc, blockID, info);
+ }
+
@Override
public String toString() {
return "ChunkLayout:v" + version;
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java
index 00627ff..ba34a29 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java
@@ -24,6 +24,7 @@
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.List;
+
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.
ContainerType;
@@ -504,6 +505,15 @@
}
/**
+ * Decrease the count of keys in the container.
+ *
+ * @param deletedKeyCount
+ */
+ public void decrKeyCount(long deletedKeyCount) {
+ this.keyCount.addAndGet(-1 * deletedKeyCount);
+ }
+
+ /**
* Returns number of keys in the container.
* @return key count
*/
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
index cef1c8f..c998f89 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
@@ -18,23 +18,28 @@
package org.apache.hadoop.ozone.container.common.impl;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerDataProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerAction;
-import org.apache.hadoop.hdds.scm.container.common.helpers
- .ContainerNotOpenException;
-import org.apache.hadoop.hdds.scm.container.common.helpers
- .InvalidContainerStateException;
-import org.apache.hadoop.hdds.scm.container.common.helpers
- .StorageContainerException;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerType;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerAction;
+import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerNotOpenException;
+import org.apache.hadoop.hdds.scm.container.common.helpers.InvalidContainerStateException;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.hdds.security.token.TokenVerifier;
import org.apache.hadoop.hdds.tracing.TracingUtil;
+import org.apache.hadoop.hdds.utils.ProtocolMessageMetrics;
import org.apache.hadoop.ozone.audit.AuditAction;
import org.apache.hadoop.ozone.audit.AuditEventStatus;
import org.apache.hadoop.ozone.audit.AuditLogger;
@@ -42,40 +47,27 @@
import org.apache.hadoop.ozone.audit.AuditMarker;
import org.apache.hadoop.ozone.audit.AuditMessage;
import org.apache.hadoop.ozone.audit.Auditor;
-import org.apache.hadoop.ozone.container.common.helpers
- .ContainerCommandRequestPBHelper;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerCommandRequestPBHelper;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
import org.apache.hadoop.ozone.container.common.interfaces.Handler;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.container.common.transport.server.ratis
- .DispatcherContext;
+import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerCommandRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerCommandResponseProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerType;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.
- ContainerDataProto.State;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-
-import io.opentracing.Scope;
import org.apache.hadoop.security.UserGroupInformation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import io.opentracing.Scope;
+import io.opentracing.Span;
+import io.opentracing.util.GlobalTracer;
import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.malformedRequest;
import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.unsupportedRequest;
+import org.apache.ratis.thirdparty.com.google.protobuf.ProtocolMessageEnum;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Ozone Container dispatcher takes a call from the netty server and routes it
@@ -87,11 +79,12 @@
private static final AuditLogger AUDIT =
new AuditLogger(AuditLoggerType.DNLOGGER);
private final Map<ContainerType, Handler> handlers;
- private final Configuration conf;
+ private final ConfigurationSource conf;
private final ContainerSet containerSet;
private final VolumeSet volumeSet;
private final StateContext context;
private final float containerCloseThreshold;
+ private final ProtocolMessageMetrics<ProtocolMessageEnum> protocolMetrics;
private String scmID;
private ContainerMetrics metrics;
private final TokenVerifier tokenVerifier;
@@ -101,7 +94,7 @@
* Constructs an OzoneContainer that receives calls from
* XceiverServerHandler.
*/
- public HddsDispatcher(Configuration config, ContainerSet contSet,
+ public HddsDispatcher(ConfigurationSource config, ContainerSet contSet,
VolumeSet volumes, Map<ContainerType, Handler> handlers,
StateContext context, ContainerMetrics metrics,
TokenVerifier tokenVerifier) {
@@ -118,14 +111,22 @@
HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED,
HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED_DEFAULT);
this.tokenVerifier = tokenVerifier;
+
+ protocolMetrics =
+ new ProtocolMessageMetrics<ProtocolMessageEnum>(
+ "HddsDispatcher",
+ "HDDS dispatcher metrics",
+ ContainerProtos.Type.values());
}
@Override
public void init() {
+ protocolMetrics.register();
}
@Override
public void shutdown() {
+ protocolMetrics.unregister();
}
/**
@@ -157,9 +158,15 @@
public ContainerCommandResponseProto dispatch(
ContainerCommandRequestProto msg, DispatcherContext dispatcherContext) {
String spanName = "HddsDispatcher." + msg.getCmdType().name();
- try (Scope scope = TracingUtil
- .importAndCreateScope(spanName, msg.getTraceID())) {
+ long startTime = System.nanoTime();
+ Span span = TracingUtil
+ .importAndCreateSpan(spanName, msg.getTraceID());
+ try (Scope scope = GlobalTracer.get().activateSpan(span)) {
return dispatchRequest(msg, dispatcherContext);
+ } finally {
+ span.finish();
+ protocolMetrics
+ .increment(msg.getCmdType(), System.nanoTime() - startTime);
}
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
index 9f5b9f7..4ba7572 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
@@ -23,7 +23,7 @@
import java.io.OutputStream;
import java.util.function.Consumer;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerType;
@@ -45,7 +45,7 @@
@SuppressWarnings("visibilitymodifier")
public abstract class Handler {
- protected final Configuration conf;
+ protected final ConfigurationSource conf;
protected final ContainerSet containerSet;
protected final VolumeSet volumeSet;
protected String scmID;
@@ -53,7 +53,7 @@
protected String datanodeId;
private Consumer<ContainerReplicaProto> icrSender;
- protected Handler(Configuration config, String datanodeId,
+ protected Handler(ConfigurationSource config, String datanodeId,
ContainerSet contSet, VolumeSet volumeSet,
ContainerMetrics containerMetrics,
Consumer<ContainerReplicaProto> icrSender) {
@@ -66,7 +66,7 @@
}
public static Handler getHandlerForContainerType(
- final ContainerType containerType, final Configuration config,
+ final ContainerType containerType, final ConfigurationSource config,
final String datanodeId, final ContainerSet contSet,
final VolumeSet volumeSet, final ContainerMetrics metrics,
Consumer<ContainerReplicaProto> icrSender) {
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportManager.java
index 536d4cc..bb43d0f 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportManager.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportManager.java
@@ -17,20 +17,21 @@
package org.apache.hadoop.ozone.container.common.report;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.google.protobuf.GeneratedMessage;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.util.concurrent.HadoopExecutors;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
+
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.protobuf.GeneratedMessage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* ReportManager is responsible for managing all the {@link ReportPublisher}
* and also provides {@link ScheduledExecutorService} to ReportPublisher
@@ -89,7 +90,7 @@
* @param conf - Conf
* @return builder - Builder.
*/
- public static Builder newBuilder(Configuration conf) {
+ public static Builder newBuilder(ConfigurationSource conf) {
return new Builder(conf);
}
@@ -103,7 +104,7 @@
private ReportPublisherFactory publisherFactory;
- private Builder(Configuration conf) {
+ private Builder(ConfigurationSource conf) {
this.reportPublishers = new ArrayList<>();
this.publisherFactory = new ReportPublisherFactory(conf);
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisher.java
index a5e04aa..685a1d9 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisher.java
@@ -17,30 +17,29 @@
package org.apache.hadoop.ozone.container.common.report;
-import com.google.protobuf.GeneratedMessage;
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.container.common.statemachine
- .DatanodeStateMachine.DatanodeStates;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.IOException;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.DatanodeStates;
+import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+
+import com.google.protobuf.GeneratedMessage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* Abstract class responsible for scheduling the reports based on the
* configured interval. All the ReportPublishers should extend this class.
*/
public abstract class ReportPublisher<T extends GeneratedMessage>
- implements Configurable, Runnable {
+ implements Runnable {
private static final Logger LOG = LoggerFactory.getLogger(
ReportPublisher.class);
- private Configuration config;
+ private ConfigurationSource config;
private StateContext context;
private ScheduledExecutorService executor;
@@ -58,13 +57,11 @@
getReportFrequency(), TimeUnit.MILLISECONDS);
}
- @Override
- public void setConf(Configuration conf) {
+ public void setConf(ConfigurationSource conf) {
config = conf;
}
- @Override
- public Configuration getConf() {
+ public ConfigurationSource getConf() {
return config;
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java
index 1c456a0..4533691 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java
@@ -17,27 +17,23 @@
package org.apache.hadoop.ozone.container.common.report;
-import com.google.protobuf.GeneratedMessage;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.protocol.proto.
- StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
-import org.apache.hadoop.hdds.protocol.proto.
- StorageContainerDatanodeProtocolProtos.CommandStatusReportsProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.NodeReportProto;
-import org.apache.hadoop.util.ReflectionUtils;
-
import java.util.HashMap;
import java.util.Map;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandStatusReportsProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+
+import com.google.protobuf.GeneratedMessage;
+
/**
* Factory class to construct {@link ReportPublisher} for a report.
*/
public class ReportPublisherFactory {
- private final Configuration conf;
+ private final ConfigurationSource conf;
private final Map<Class<? extends GeneratedMessage>,
Class<? extends ReportPublisher>> report2publisher;
@@ -46,7 +42,7 @@
*
* @param conf Configuration to be passed to the {@link ReportPublisher}
*/
- public ReportPublisherFactory(Configuration conf) {
+ public ReportPublisherFactory(ConfigurationSource conf) {
this.conf = conf;
this.report2publisher = new HashMap<>();
@@ -73,7 +69,13 @@
if (publisherClass == null) {
throw new RuntimeException("No publisher found for report " + report);
}
- return ReflectionUtils.newInstance(publisherClass, conf);
+ try {
+ ReportPublisher reportPublisher = publisherClass.newInstance();
+ reportPublisher.setConf(conf);
+ return reportPublisher;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
}
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
index dcde6fe..e41a537 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
@@ -24,34 +24,23 @@
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.CommandStatusReportsProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.NodeReportProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandStatusReportsProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
+import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
import org.apache.hadoop.ozone.HddsDatanodeStopService;
import org.apache.hadoop.ozone.container.common.report.ReportManager;
-import org.apache.hadoop.ozone.container.common.statemachine.commandhandler
- .CloseContainerCommandHandler;
-import org.apache.hadoop.ozone.container.common.statemachine.commandhandler
- .ClosePipelineCommandHandler;
-import org.apache.hadoop.ozone.container.common.statemachine.commandhandler
- .CommandDispatcher;
-import org.apache.hadoop.ozone.container.common.statemachine.commandhandler
- .CreatePipelineCommandHandler;
-import org.apache.hadoop.ozone.container.common.statemachine.commandhandler
- .DeleteBlocksCommandHandler;
-import org.apache.hadoop.ozone.container.common.statemachine.commandhandler
- .DeleteContainerCommandHandler;
-import org.apache.hadoop.ozone.container.common.statemachine.commandhandler
- .ReplicateContainerCommandHandler;
+import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.CloseContainerCommandHandler;
+import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.ClosePipelineCommandHandler;
+import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.CommandDispatcher;
+import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.CreatePipelineCommandHandler;
+import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.DeleteBlocksCommandHandler;
+import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.DeleteContainerCommandHandler;
+import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.ReplicateContainerCommandHandler;
import org.apache.hadoop.ozone.container.keyvalue.TarContainerPacker;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.container.replication.ContainerReplicator;
@@ -76,7 +65,7 @@
static final Logger LOG =
LoggerFactory.getLogger(DatanodeStateMachine.class);
private final ExecutorService executorService;
- private final Configuration conf;
+ private final ConfigurationSource conf;
private final SCMConnectionManager connectionManager;
private StateContext context;
private final OzoneContainer container;
@@ -106,11 +95,10 @@
* enabled
*/
public DatanodeStateMachine(DatanodeDetails datanodeDetails,
- Configuration conf, CertificateClient certClient,
+ ConfigurationSource conf, CertificateClient certClient,
HddsDatanodeStopService hddsDatanodeStopService) throws IOException {
- OzoneConfiguration ozoneConf = new OzoneConfiguration(conf);
DatanodeConfiguration dnConf =
- ozoneConf.getObject(DatanodeConfiguration.class);
+ conf.getObject(DatanodeConfiguration.class);
this.hddsDatanodeStopService = hddsDatanodeStopService;
this.conf = conf;
@@ -126,7 +114,7 @@
constructionLock.writeLock().lock();
try {
container = new OzoneContainer(this.datanodeDetails,
- ozoneConf, context, certClient);
+ conf, context, certClient);
} finally {
constructionLock.writeLock().unlock();
}
@@ -208,7 +196,8 @@
// Start jvm monitor
jvmPauseMonitor = new JvmPauseMonitor();
- jvmPauseMonitor.init(conf);
+ jvmPauseMonitor
+ .init(LegacyHadoopConfigurationSource.asHadoopConfiguration(conf));
jvmPauseMonitor.start();
while (context.getState() != DatanodeStates.SHUTDOWN) {
@@ -456,7 +445,7 @@
*
* @param config
*/
- private void initCommandHandlerThread(Configuration config) {
+ private void initCommandHandlerThread(ConfigurationSource config) {
/**
* Task that periodically checks if we have any outstanding commands.
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachine.java
index a500d4a..cd1a376 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachine.java
@@ -16,7 +16,7 @@
*/
package org.apache.hadoop.ozone.container.common.statemachine;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.ozone.protocol.VersionResponse;
import org.apache.hadoop.ozone.protocolPB
.StorageContainerDatanodeProtocolClientSideTranslatorPB;
@@ -46,7 +46,7 @@
private final AtomicLong missedCount;
private final InetSocketAddress address;
private final Lock lock;
- private final Configuration conf;
+ private final ConfigurationSource conf;
private EndPointStates state;
private VersionResponse version;
private ZonedDateTime lastSuccessfulHeartbeat;
@@ -59,7 +59,7 @@
*/
public EndpointStateMachine(InetSocketAddress address,
StorageContainerDatanodeProtocolClientSideTranslatorPB endPoint,
- Configuration conf) {
+ ConfigurationSource conf) {
this.endPoint = endPoint;
this.missedCount = new AtomicLong(0);
this.address = address;
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManager.java
index 814eeb4..ebc53c9 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManager.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManager.java
@@ -16,22 +16,6 @@
*/
package org.apache.hadoop.ozone.container.common.statemachine;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.retry.RetryPolicies;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.metrics2.util.MBeans;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.ozone.protocolPB.ReconDatanodeProtocolPB;
-import org.apache.hadoop.ozone.protocolPB
- .StorageContainerDatanodeProtocolClientSideTranslatorPB;
-import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolPB;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import javax.management.ObjectName;
import java.io.Closeable;
import java.io.IOException;
@@ -45,9 +29,25 @@
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.retry.RetryPolicies;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.metrics2.util.MBeans;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.ozone.protocolPB.ReconDatanodeProtocolPB;
+import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolClientSideTranslatorPB;
+import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolPB;
+import org.apache.hadoop.security.UserGroupInformation;
+
import static java.util.Collections.unmodifiableList;
-import static org.apache.hadoop.hdds.utils.HddsServerUtil
- .getScmRpcTimeOutInMilliseconds;
+import static org.apache.hadoop.hdds.utils.HddsServerUtil.getScmRpcTimeOutInMilliseconds;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* SCMConnectionManager - Acts as a class that manages the membership
@@ -62,10 +62,10 @@
private final Map<InetSocketAddress, EndpointStateMachine> scmMachines;
private final int rpcTimeout;
- private final Configuration conf;
+ private final ConfigurationSource conf;
private ObjectName jmxBean;
- public SCMConnectionManager(Configuration conf) {
+ public SCMConnectionManager(ConfigurationSource conf) {
this.mapLock = new ReentrantReadWriteLock();
Long timeOut = getScmRpcTimeOutInMilliseconds(conf);
this.rpcTimeout = timeOut.intValue();
@@ -82,7 +82,7 @@
*
* @return ozoneConfig.
*/
- public Configuration getConf() {
+ public ConfigurationSource getConf() {
return conf;
}
@@ -139,7 +139,11 @@
return;
}
- RPC.setProtocolEngine(conf, StorageContainerDatanodeProtocolPB.class,
+ Configuration hadoopConfig =
+ LegacyHadoopConfigurationSource.asHadoopConfiguration(this.conf);
+ RPC.setProtocolEngine(
+ hadoopConfig,
+ StorageContainerDatanodeProtocolPB.class,
ProtobufRpcEngine.class);
long version =
RPC.getProtocolVersion(StorageContainerDatanodeProtocolPB.class);
@@ -150,8 +154,8 @@
StorageContainerDatanodeProtocolPB rpcProxy = RPC.getProtocolProxy(
StorageContainerDatanodeProtocolPB.class, version,
- address, UserGroupInformation.getCurrentUser(), conf,
- NetUtils.getDefaultSocketFactory(conf), getRpcTimeout(),
+ address, UserGroupInformation.getCurrentUser(), hadoopConfig,
+ NetUtils.getDefaultSocketFactory(hadoopConfig), getRpcTimeout(),
retryPolicy).getProxy();
StorageContainerDatanodeProtocolClientSideTranslatorPB rpcClient =
@@ -159,7 +163,7 @@
rpcProxy);
EndpointStateMachine endPoint =
- new EndpointStateMachine(address, rpcClient, conf);
+ new EndpointStateMachine(address, rpcClient, this.conf);
endPoint.setPassive(false);
scmMachines.put(address, endPoint);
} finally {
@@ -181,8 +185,9 @@
"Ignoring the request.");
return;
}
-
- RPC.setProtocolEngine(conf, ReconDatanodeProtocolPB.class,
+ Configuration hadoopConfig =
+ LegacyHadoopConfigurationSource.asHadoopConfiguration(this.conf);
+ RPC.setProtocolEngine(hadoopConfig, ReconDatanodeProtocolPB.class,
ProtobufRpcEngine.class);
long version =
RPC.getProtocolVersion(ReconDatanodeProtocolPB.class);
@@ -192,8 +197,8 @@
60000, TimeUnit.MILLISECONDS);
ReconDatanodeProtocolPB rpcProxy = RPC.getProtocolProxy(
ReconDatanodeProtocolPB.class, version,
- address, UserGroupInformation.getCurrentUser(), conf,
- NetUtils.getDefaultSocketFactory(conf), getRpcTimeout(),
+ address, UserGroupInformation.getCurrentUser(), hadoopConfig,
+ NetUtils.getDefaultSocketFactory(hadoopConfig), getRpcTimeout(),
retryPolicy).getProxy();
StorageContainerDatanodeProtocolClientSideTranslatorPB rpcClient =
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
index 04502b6..73bea25 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
@@ -16,45 +16,16 @@
*/
package org.apache.hadoop.ozone.container.common.statemachine;
-import com.google.common.base.Preconditions;
-import com.google.protobuf.GeneratedMessage;
-
import java.net.InetSocketAddress;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMCommandProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.PipelineAction;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerAction;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.CommandStatus.Status;
-import org.apache.hadoop.ozone.container.common.states.DatanodeState;
-import org.apache.hadoop.ozone.container.common.states.datanode
- .InitDatanodeState;
-import org.apache.hadoop.ozone.container.common.states.datanode
- .RunningDatanodeState;
-import org.apache.hadoop.ozone.protocol.commands.CommandStatus;
-import org.apache.hadoop.ozone.protocol.commands
- .DeleteBlockCommandStatus.DeleteBlockCommandStatusBuilder;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-
-import static java.lang.Math.min;
-import static org.apache.hadoop.hdds.utils.HddsServerUtil.getScmHeartbeatInterval;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
import java.util.Queue;
-import java.util.ArrayList;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
@@ -64,6 +35,26 @@
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Consumer;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandStatus.Status;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerAction;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineAction;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
+import org.apache.hadoop.ozone.container.common.states.DatanodeState;
+import org.apache.hadoop.ozone.container.common.states.datanode.InitDatanodeState;
+import org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState;
+import org.apache.hadoop.ozone.protocol.commands.CommandStatus;
+import org.apache.hadoop.ozone.protocol.commands.DeleteBlockCommandStatus.DeleteBlockCommandStatusBuilder;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+
+import com.google.common.base.Preconditions;
+import com.google.protobuf.GeneratedMessage;
+import static java.lang.Math.min;
+import org.apache.commons.collections.CollectionUtils;
+import static org.apache.hadoop.hdds.utils.HddsServerUtil.getScmHeartbeatInterval;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* Current Context of State Machine.
*/
@@ -75,7 +66,7 @@
private final Lock lock;
private final DatanodeStateMachine parent;
private final AtomicLong stateExecutionCount;
- private final Configuration conf;
+ private final ConfigurationSource conf;
private final Set<InetSocketAddress> endpoints;
private final Map<InetSocketAddress, List<GeneratedMessage>> reports;
private final Map<InetSocketAddress, Queue<ContainerAction>> containerActions;
@@ -98,8 +89,9 @@
* @param state - State
* @param parent Parent State Machine
*/
- public StateContext(Configuration conf, DatanodeStateMachine.DatanodeStates
- state, DatanodeStateMachine parent) {
+ public StateContext(ConfigurationSource conf,
+ DatanodeStateMachine.DatanodeStates
+ state, DatanodeStateMachine parent) {
this.conf = conf;
this.state = state;
this.parent = parent;
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CreatePipelineCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CreatePipelineCommandHandler.java
index 49a8fd9..c60c112 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CreatePipelineCommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CreatePipelineCommandHandler.java
@@ -16,24 +16,26 @@
*/
package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
-import org.apache.hadoop.conf.Configuration;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto.
- StorageContainerDatanodeProtocolProtos.CreatePipelineCommandProto;
-import org.apache.hadoop.hdds.protocol.proto.
- StorageContainerDatanodeProtocolProtos.SCMCommandProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CreatePipelineCommandProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import org.apache.hadoop.hdds.ratis.RatisHelper;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
-import org.apache.hadoop.ozone.container.common.statemachine
- .SCMConnectionManager;
+import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.container.common.transport.server
- .XceiverServerSpi;
+import org.apache.hadoop.ozone.container.common.transport.server.XceiverServerSpi;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.protocol.commands.CreatePipelineCommand;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.hadoop.util.Time;
+
import org.apache.ratis.client.RaftClient;
import org.apache.ratis.protocol.AlreadyExistsException;
import org.apache.ratis.protocol.RaftGroup;
@@ -42,11 +44,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.stream.Collectors;
-
/**
* Handler for create pipeline command received from SCM.
*/
@@ -56,14 +53,14 @@
LoggerFactory.getLogger(CreatePipelineCommandHandler.class);
private final AtomicLong invocationCount = new AtomicLong(0);
- private final Configuration conf;
+ private final ConfigurationSource conf;
private long totalTime;
/**
* Constructs a createPipelineCommand handler.
*/
- public CreatePipelineCommandHandler(Configuration conf) {
+ public CreatePipelineCommandHandler(ConfigurationSource conf) {
this.conf = conf;
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
index eac26f1..4324875 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
@@ -17,7 +17,7 @@
package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
import com.google.common.primitives.Longs;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
@@ -58,6 +58,8 @@
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.Result.CONTAINER_NOT_FOUND;
+import static org.apache.hadoop.ozone.OzoneConsts.DB_CONTAINER_DELETE_TRANSACTION_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.DB_PENDING_DELETE_BLOCK_COUNT_KEY;
/**
* Handle block deletion commands.
@@ -68,13 +70,13 @@
LoggerFactory.getLogger(DeleteBlocksCommandHandler.class);
private final ContainerSet containerSet;
- private final Configuration conf;
+ private final ConfigurationSource conf;
private int invocationCount;
private long totalTime;
private boolean cmdExecuted;
public DeleteBlocksCommandHandler(ContainerSet cset,
- Configuration conf) {
+ ConfigurationSource conf) {
this.containerSet = cset;
this.conf = conf;
}
@@ -251,12 +253,27 @@
}
}
- containerDB.getStore()
- .put(DFSUtil.string2Bytes(OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX),
- Longs.toByteArray(delTX.getTxID()));
- containerData
- .updateDeleteTransactionId(delTX.getTxID());
- // update pending deletion blocks count in in-memory container status
+ // Finally commit the DB counters.
+ BatchOperation batchOperation = new BatchOperation();
+
+ // In memory is updated only when existing delete transactionID is
+ // greater.
+ if (delTX.getTxID() > containerData.getDeleteTransactionId()) {
+ // Update in DB pending delete key count and delete transaction ID.
+ batchOperation.put(DB_CONTAINER_DELETE_TRANSACTION_KEY,
+ Longs.toByteArray(delTX.getTxID()));
+ }
+
+ batchOperation.put(DB_PENDING_DELETE_BLOCK_COUNT_KEY, Longs.toByteArray(
+ containerData.getNumPendingDeletionBlocks() + newDeletionBlocks));
+
+ containerDB.getStore().writeBatch(batchOperation);
+
+
+ // update pending deletion blocks count and delete transaction ID in
+ // in-memory container status
+ containerData.updateDeleteTransactionId(delTX.getTxID());
+
containerData.incrPendingDeletionBlocks(newDeletionBlocks);
}
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
index a96032e..41958bf 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
@@ -18,7 +18,7 @@
import java.util.List;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type;
@@ -46,12 +46,12 @@
private long totalTime;
- private Configuration conf;
+ private ConfigurationSource conf;
private ReplicationSupervisor supervisor;
public ReplicateContainerCommandHandler(
- Configuration conf,
+ ConfigurationSource conf,
ReplicationSupervisor supervisor) {
this.conf = conf;
this.supervisor = supervisor;
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java
index a73f1c5..ba898db 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java
@@ -16,21 +16,6 @@
*/
package org.apache.hadoop.ozone.container.common.states.datanode;
-import com.google.common.base.Strings;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.utils.HddsServerUtil;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.hadoop.ozone.container.common.statemachine
- .DatanodeStateMachine;
-import org.apache.hadoop.ozone.container.common.statemachine
- .SCMConnectionManager;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.container.common.states.DatanodeState;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -42,8 +27,21 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.utils.HddsServerUtil;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
+import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
+import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+import org.apache.hadoop.ozone.container.common.states.DatanodeState;
+
+import com.google.common.base.Strings;
import static org.apache.hadoop.hdds.HddsUtils.getReconAddresses;
import static org.apache.hadoop.hdds.HddsUtils.getSCMAddresses;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Init Datanode State is the task that gets run when we are in Init State.
@@ -52,7 +50,7 @@
Callable<DatanodeStateMachine.DatanodeStates> {
static final Logger LOG = LoggerFactory.getLogger(InitDatanodeState.class);
private final SCMConnectionManager connectionManager;
- private final Configuration conf;
+ private final ConfigurationSource conf;
private final StateContext context;
private Future<DatanodeStateMachine.DatanodeStates> result;
@@ -63,7 +61,7 @@
* @param connectionManager - Connection Manager
* @param context - Current Context
*/
- public InitDatanodeState(Configuration conf,
+ public InitDatanodeState(ConfigurationSource conf,
SCMConnectionManager connectionManager,
StateContext context) {
this.conf = conf;
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java
index 779b1a2..1ecfbf9 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java
@@ -16,7 +16,7 @@
*/
package org.apache.hadoop.ozone.container.common.states.datanode;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine.EndPointStates;
@@ -51,14 +51,14 @@
static final Logger
LOG = LoggerFactory.getLogger(RunningDatanodeState.class);
private final SCMConnectionManager connectionManager;
- private final Configuration conf;
+ private final ConfigurationSource conf;
private final StateContext context;
private CompletionService<EndPointStates> ecs;
/** Cache the end point task per end point per end point state. */
private Map<EndpointStateMachine, Map<EndPointStates,
Callable<EndPointStates>>> endpointTasks;
- public RunningDatanodeState(Configuration conf,
+ public RunningDatanodeState(ConfigurationSource conf,
SCMConnectionManager connectionManager,
StateContext context) {
this.connectionManager = connectionManager;
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
index fb1d1af..494ccd9 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
@@ -21,7 +21,7 @@
import com.google.common.base.Preconditions;
import com.google.protobuf.Descriptors;
import com.google.protobuf.GeneratedMessage;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
import org.apache.hadoop.hdds.protocol.proto
@@ -78,7 +78,7 @@
static final Logger LOG =
LoggerFactory.getLogger(HeartbeatEndpointTask.class);
private final EndpointStateMachine rpcEndpoint;
- private final Configuration conf;
+ private final ConfigurationSource conf;
private DatanodeDetailsProto datanodeDetailsProto;
private StateContext context;
private int maxContainerActionsPerHB;
@@ -90,7 +90,7 @@
* @param conf Config.
*/
public HeartbeatEndpointTask(EndpointStateMachine rpcEndpoint,
- Configuration conf, StateContext context) {
+ ConfigurationSource conf, StateContext context) {
this.rpcEndpoint = rpcEndpoint;
this.conf = conf;
this.context = context;
@@ -344,7 +344,7 @@
*/
public static class Builder {
private EndpointStateMachine endPointStateMachine;
- private Configuration conf;
+ private ConfigurationSource conf;
private DatanodeDetails datanodeDetails;
private StateContext context;
@@ -371,7 +371,7 @@
* @param config - config
* @return Builder
*/
- public Builder setConfig(Configuration config) {
+ public Builder setConfig(ConfigurationSource config) {
this.conf = config;
return this;
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
index 92e5743..be95f01 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
@@ -19,7 +19,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
@@ -49,7 +49,7 @@
static final Logger LOG = LoggerFactory.getLogger(RegisterEndpointTask.class);
private final EndpointStateMachine rpcEndPoint;
- private final Configuration conf;
+ private final ConfigurationSource conf;
private Future<EndpointStateMachine.EndPointStates> result;
private DatanodeDetails datanodeDetails;
private final OzoneContainer datanodeContainerManager;
@@ -64,7 +64,7 @@
*/
@VisibleForTesting
public RegisterEndpointTask(EndpointStateMachine rpcEndPoint,
- Configuration conf, OzoneContainer ozoneContainer,
+ ConfigurationSource conf, OzoneContainer ozoneContainer,
StateContext context) {
this.rpcEndPoint = rpcEndPoint;
this.conf = conf;
@@ -163,7 +163,7 @@
*/
public static class Builder {
private EndpointStateMachine endPointStateMachine;
- private Configuration conf;
+ private ConfigurationSource conf;
private DatanodeDetails datanodeDetails;
private OzoneContainer container;
private StateContext context;
@@ -191,7 +191,7 @@
* @param config - config
* @return Builder.
*/
- public Builder setConfig(Configuration config) {
+ public Builder setConfig(ConfigurationSource config) {
this.conf = config;
return this;
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java
index 0834f77..6c53756 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java
@@ -16,26 +16,25 @@
*/
package org.apache.hadoop.ozone.container.common.states.endpoint;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto;
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.Callable;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto;
import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.container.common.statemachine
- .EndpointStateMachine;
+import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine;
import org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.protocol.VersionResponse;
import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
+
+import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.Map;
-import java.util.concurrent.Callable;
-
/**
* Task that returns version.
*/
@@ -44,11 +43,11 @@
public static final Logger LOG = LoggerFactory.getLogger(VersionEndpointTask
.class);
private final EndpointStateMachine rpcEndPoint;
- private final Configuration configuration;
+ private final ConfigurationSource configuration;
private final OzoneContainer ozoneContainer;
public VersionEndpointTask(EndpointStateMachine rpcEndPoint,
- Configuration conf, OzoneContainer container) {
+ ConfigurationSource conf, OzoneContainer container) {
this.rpcEndPoint = rpcEndPoint;
this.configuration = conf;
this.ozoneContainer = container;
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
index 441d9c8..3647af1 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
@@ -18,19 +18,21 @@
package org.apache.hadoop.ozone.container.common.transport.server;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerCommandRequestProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.PipelineReport;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReport;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
-import org.apache.hadoop.hdds.scm.container.common.helpers.
- StorageContainerException;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.hdds.tracing.GrpcServerInterceptor;
@@ -39,7 +41,10 @@
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
+import com.google.common.base.Preconditions;
import io.opentracing.Scope;
+import io.opentracing.Span;
+import io.opentracing.util.GlobalTracer;
import org.apache.ratis.thirdparty.io.grpc.BindableService;
import org.apache.ratis.thirdparty.io.grpc.Server;
import org.apache.ratis.thirdparty.io.grpc.ServerBuilder;
@@ -50,12 +55,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-
/**
* Creates a Grpc server endpoint that acts as the communication layer for
* Ozone containers.
@@ -77,7 +76,8 @@
*
* @param conf - Configuration
*/
- public XceiverServerGrpc(DatanodeDetails datanodeDetails, Configuration conf,
+ public XceiverServerGrpc(DatanodeDetails datanodeDetails,
+ ConfigurationSource conf,
ContainerDispatcher dispatcher, CertificateClient caClient,
BindableService... additionalServices) {
Preconditions.checkNotNull(conf);
@@ -172,16 +172,19 @@
@Override
public void submitRequest(ContainerCommandRequestProto request,
HddsProtos.PipelineID pipelineID) throws IOException {
- try (Scope scope = TracingUtil
- .importAndCreateScope(
+ Span span = TracingUtil
+ .importAndCreateSpan(
"XceiverServerGrpc." + request.getCmdType().name(),
- request.getTraceID())) {
+ request.getTraceID());
+ try (Scope scope = GlobalTracer.get().activateSpan(span)) {
ContainerProtos.ContainerCommandResponseProto response =
storageContainer.dispatch(request, null);
if (response.getResult() != ContainerProtos.Result.SUCCESS) {
throw new StorageContainerException(response.getMessage(),
response.getResult());
}
+ } finally {
+ span.finish();
}
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
index 9311238..c529e7b 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
@@ -18,14 +18,36 @@
package org.apache.hadoop.ozone.container.common.transport.server.ratis;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.StorageUnit;
-import org.apache.hadoop.hdds.HddsUtils;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+import org.apache.hadoop.hdds.HddsUtils;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Container2BCSIDMapProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ReadChunkRequestProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ReadChunkResponseProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.WriteChunkRequestProto;
import org.apache.hadoop.hdds.ratis.ContainerCommandRequestMessage;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerNotOpenException;
@@ -33,11 +55,18 @@
import org.apache.hadoop.hdds.utils.Cache;
import org.apache.hadoop.hdds.utils.ResourceLimitCache;
import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig;
+import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
import org.apache.hadoop.util.Time;
-import org.apache.ratis.proto.RaftProtos.StateMachineLogEntryProto;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.ratis.proto.RaftProtos.LogEntryProto;
import org.apache.ratis.proto.RaftProtos.RaftPeerRole;
+import org.apache.ratis.proto.RaftProtos.RoleInfoProto;
+import org.apache.ratis.proto.RaftProtos.StateMachineLogEntryProto;
+import org.apache.ratis.protocol.Message;
+import org.apache.ratis.protocol.RaftClientRequest;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.protocol.RaftGroupMemberId;
import org.apache.ratis.protocol.RaftPeerId;
@@ -46,54 +75,18 @@
import org.apache.ratis.server.impl.RaftServerProxy;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.server.raftlog.RaftLog;
-import org.apache.ratis.statemachine.impl.SingleFileSnapshotInfo;
-import org.apache.ratis.thirdparty.com.google.protobuf
- .InvalidProtocolBufferException;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.
- Container2BCSIDMapProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerCommandRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerCommandResponseProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .WriteChunkRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ReadChunkRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ReadChunkResponseProto;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.ratis.protocol.Message;
-import org.apache.ratis.protocol.RaftClientRequest;
import org.apache.ratis.server.storage.RaftStorage;
-import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
-import org.apache.ratis.proto.RaftProtos.RoleInfoProto;
-import org.apache.ratis.proto.RaftProtos.LogEntryProto;
import org.apache.ratis.statemachine.StateMachineStorage;
import org.apache.ratis.statemachine.TransactionContext;
import org.apache.ratis.statemachine.impl.BaseStateMachine;
import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage;
+import org.apache.ratis.statemachine.impl.SingleFileSnapshotInfo;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
import org.apache.ratis.thirdparty.com.google.protobuf.TextFormat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.stream.Collectors;
-import java.util.concurrent.Executors;
-import java.io.FileOutputStream;
-import java.io.FileInputStream;
-import java.io.OutputStream;
-
/** A {@link org.apache.ratis.statemachine.StateMachine} for containers.
*
* The stateMachine is responsible for handling different types of container
@@ -161,7 +154,7 @@
public ContainerStateMachine(RaftGroupId gid, ContainerDispatcher dispatcher,
ContainerController containerController,
List<ThreadPoolExecutor> chunkExecutors,
- XceiverServerRatis ratisServer, Configuration conf) {
+ XceiverServerRatis ratisServer, ConfigurationSource conf) {
this.gid = gid;
this.dispatcher = dispatcher;
this.containerController = containerController;
@@ -609,19 +602,6 @@
}
/**
- * Reads the Entry from the Cache or loads it back by reading from disk.
- */
- private ByteString getCachedStateMachineData(Long logIndex, long term,
- ContainerCommandRequestProto requestProto)
- throws IOException {
- ByteString data = stateMachineDataCache.get(logIndex);
- if (data == null) {
- data = readStateMachineData(requestProto, term, logIndex);
- }
- return data;
- }
-
- /**
* Returns the combined future of all the writeChunks till the given log
* index. The Raft log worker will wait for the stateMachineData to complete
* flush as well.
@@ -659,11 +639,17 @@
Preconditions.checkArgument(!HddsUtils.isReadOnly(requestProto));
if (requestProto.getCmdType() == Type.WriteChunk) {
final CompletableFuture<ByteString> future = new CompletableFuture<>();
+ ByteString data = stateMachineDataCache.get(entry.getIndex());
+ if (data != null) {
+ future.complete(data);
+ return future;
+ }
+
CompletableFuture.supplyAsync(() -> {
try {
future.complete(
- getCachedStateMachineData(entry.getIndex(), entry.getTerm(),
- requestProto));
+ readStateMachineData(requestProto, entry.getTerm(),
+ entry.getIndex()));
} catch (IOException e) {
metrics.incNumReadStateMachineFails();
future.completeExceptionally(e);
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index b2560b7..39e23e4 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -18,69 +18,18 @@
package org.apache.hadoop.ozone.container.common.transport.server.ratis;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableList;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.StorageUnit;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReport;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ClosePipelineInfo;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineAction;
-import org.apache.hadoop.hdds.ratis.ContainerCommandRequestMessage;
-import org.apache.hadoop.hdds.utils.HddsServerUtil;
-import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
-import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
-import org.apache.hadoop.hdds.tracing.TracingUtil;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig;
-import org.apache.hadoop.ozone.container.common.impl.ContainerData;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-
-import io.opentracing.Scope;
-import org.apache.hadoop.ozone.container.common.transport.server.XceiverServerSpi;
-import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
-import org.apache.ratis.RaftConfigKeys;
-import org.apache.hadoop.hdds.ratis.RatisHelper;
-import org.apache.ratis.conf.RaftProperties;
-import org.apache.ratis.grpc.GrpcConfigKeys;
-import org.apache.ratis.grpc.GrpcFactory;
-import org.apache.ratis.grpc.GrpcTlsConfig;
-import org.apache.ratis.netty.NettyConfigKeys;
-import org.apache.ratis.protocol.*;
-import org.apache.ratis.rpc.RpcType;
-import org.apache.ratis.rpc.SupportedRpcType;
-import org.apache.ratis.server.RaftServer;
-import org.apache.ratis.server.RaftServerConfigKeys;
-import org.apache.ratis.proto.RaftProtos;
-import org.apache.ratis.proto.RaftProtos.RoleInfoProto;
-import org.apache.ratis.server.protocol.TermIndex;
-import org.apache.ratis.server.impl.RaftServerProxy;
-import org.apache.ratis.util.SizeInBytes;
-import org.apache.ratis.util.TimeDuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
-import java.util.Collections;
import java.util.Set;
import java.util.UUID;
-import java.util.ArrayList;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
@@ -90,6 +39,70 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ClosePipelineInfo;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineAction;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReport;
+import org.apache.hadoop.hdds.ratis.ContainerCommandRequestMessage;
+import org.apache.hadoop.hdds.ratis.RatisHelper;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
+import org.apache.hadoop.hdds.tracing.TracingUtil;
+import org.apache.hadoop.hdds.utils.HddsServerUtil;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.container.common.impl.ContainerData;
+import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
+import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+import org.apache.hadoop.ozone.container.common.transport.server.XceiverServerSpi;
+import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
+import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import io.opentracing.Scope;
+import io.opentracing.Span;
+import io.opentracing.util.GlobalTracer;
+import org.apache.ratis.RaftConfigKeys;
+import org.apache.ratis.conf.RaftProperties;
+import org.apache.ratis.grpc.GrpcConfigKeys;
+import org.apache.ratis.grpc.GrpcFactory;
+import org.apache.ratis.grpc.GrpcTlsConfig;
+import org.apache.ratis.netty.NettyConfigKeys;
+import org.apache.ratis.proto.RaftProtos;
+import org.apache.ratis.proto.RaftProtos.RoleInfoProto;
+import org.apache.ratis.protocol.ClientId;
+import org.apache.ratis.protocol.GroupInfoReply;
+import org.apache.ratis.protocol.GroupInfoRequest;
+import org.apache.ratis.protocol.GroupManagementRequest;
+import org.apache.ratis.protocol.NotLeaderException;
+import org.apache.ratis.protocol.RaftClientReply;
+import org.apache.ratis.protocol.RaftClientRequest;
+import org.apache.ratis.protocol.RaftGroup;
+import org.apache.ratis.protocol.RaftGroupId;
+import org.apache.ratis.protocol.RaftGroupMemberId;
+import org.apache.ratis.protocol.RaftPeerId;
+import org.apache.ratis.protocol.StateMachineException;
+import org.apache.ratis.rpc.RpcType;
+import org.apache.ratis.rpc.SupportedRpcType;
+import org.apache.ratis.server.RaftServer;
+import org.apache.ratis.server.RaftServerConfigKeys;
+import org.apache.ratis.server.impl.RaftServerProxy;
+import org.apache.ratis.server.protocol.TermIndex;
+import org.apache.ratis.util.SizeInBytes;
+import org.apache.ratis.util.TimeDuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
/**
* Creates a ratis server endpoint that acts as the communication layer for
* Ozone containers.
@@ -113,17 +126,19 @@
private long nodeFailureTimeoutMs;
private boolean isStarted = false;
private DatanodeDetails datanodeDetails;
- private final OzoneConfiguration conf;
+ private final ConfigurationSource conf;
// TODO: Remove the gids set when Ratis supports an api to query active
// pipelines
private final Set<RaftGroupId> raftGids = new HashSet<>();
private final RaftPeerId raftPeerId;
// pipelines for which I am the leader
private Map<RaftGroupId, Boolean> groupLeaderMap = new ConcurrentHashMap<>();
+ // Timeout used while calling submitRequest directly.
+ private long requestTimeout;
private XceiverServerRatis(DatanodeDetails dd, int port,
ContainerDispatcher dispatcher, ContainerController containerController,
- StateContext context, GrpcTlsConfig tlsConfig, OzoneConfiguration conf)
+ StateContext context, GrpcTlsConfig tlsConfig, ConfigurationSource conf)
throws IOException {
this.conf = conf;
Objects.requireNonNull(dd, "id == null");
@@ -144,6 +159,10 @@
builder.setParameters(GrpcFactory.newRaftParameters(tlsConfig));
}
this.server = builder.build();
+ this.requestTimeout = conf.getTimeDuration(
+ HddsConfigKeys.HDDS_DATANODE_RATIS_SERVER_REQUEST_TIMEOUT,
+ HddsConfigKeys.HDDS_DATANODE_RATIS_SERVER_REQUEST_TIMEOUT_DEFAULT,
+ TimeUnit.MILLISECONDS);
}
private ContainerStateMachine getStateMachine(RaftGroupId gid) {
@@ -188,14 +207,14 @@
setRatisLeaderElectionTimeout(properties);
// Set the maximum cache segments
- RaftServerConfigKeys.Log.setMaxCachedSegmentNum(properties, 2);
+ RaftServerConfigKeys.Log.setSegmentCacheNumMax(properties, 2);
// set the node failure timeout
setNodeFailureTimeout(properties);
// Set the ratis storage directory
String storageDir = HddsServerUtil.getOzoneDatanodeRatisDirectory(conf);
- RaftServerConfigKeys.setStorageDirs(properties,
+ RaftServerConfigKeys.setStorageDir(properties,
Collections.singletonList(new File(storageDir)));
// For grpc set the maximum message size
@@ -359,11 +378,11 @@
OzoneConfigKeys.DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT_DEFAULT,
StorageUnit.BYTES);
RaftServerConfigKeys.Write.setByteLimit(properties,
- pendingRequestsByteLimit);
+ SizeInBytes.valueOf(pendingRequestsByteLimit));
}
public static XceiverServerRatis newXceiverServerRatis(
- DatanodeDetails datanodeDetails, OzoneConfiguration ozoneConf,
+ DatanodeDetails datanodeDetails, ConfigurationSource ozoneConf,
ContainerDispatcher dispatcher, ContainerController containerController,
CertificateClient caClient, StateContext context) throws IOException {
int localPort = ozoneConf.getInt(
@@ -486,20 +505,24 @@
public void submitRequest(ContainerCommandRequestProto request,
HddsProtos.PipelineID pipelineID) throws IOException {
RaftClientReply reply;
- try (Scope scope = TracingUtil
- .importAndCreateScope(
+ Span span = TracingUtil
+ .importAndCreateSpan(
"XceiverServerRatis." + request.getCmdType().name(),
- request.getTraceID())) {
+ request.getTraceID());
+ try (Scope scope = GlobalTracer.get().activateSpan(span)) {
RaftClientRequest raftClientRequest =
createRaftClientRequest(request, pipelineID,
RaftClientRequest.writeRequestType());
try {
- reply = server.submitClientRequestAsync(raftClientRequest).get();
+ reply = server.submitClientRequestAsync(raftClientRequest)
+ .get(requestTimeout, TimeUnit.MILLISECONDS);
} catch (Exception e) {
throw new IOException(e.getMessage(), e);
}
processReply(reply);
+ } finally {
+ span.finish();
}
}
@@ -754,12 +777,17 @@
}
private static List<ThreadPoolExecutor> createChunkExecutors(
- Configuration conf) {
+ ConfigurationSource conf) {
// TODO create single pool with N threads if using non-incremental chunks
- final int threadCount = conf.getInt(
+ final int threadCountPerDisk = conf.getInt(
OzoneConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_KEY,
OzoneConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT);
- ThreadPoolExecutor[] executors = new ThreadPoolExecutor[threadCount];
+
+ final int numberOfDisks =
+ MutableVolumeSet.getDatanodeStorageDirs(conf).size();
+
+ ThreadPoolExecutor[] executors =
+ new ThreadPoolExecutor[threadCountPerDisk * numberOfDisks];
for (int i = 0; i < executors.length; i++) {
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setDaemon(true)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java
index 4ddb4e4..d2d2901 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java
@@ -18,21 +18,22 @@
package org.apache.hadoop.ozone.container.common.utils;
-import com.google.common.base.Preconditions;
-import org.apache.commons.collections.MapIterator;
-import org.apache.commons.collections.map.LRUMap;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.hdds.utils.MetadataStore;
-import org.apache.hadoop.hdds.utils.MetadataStoreBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.File;
import java.io.IOException;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.utils.MetadataStore;
+import org.apache.hadoop.hdds.utils.MetadataStoreBuilder;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.collections.MapIterator;
+import org.apache.commons.collections.map.LRUMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* container cache is a LRUMap that maintains the DB handles.
*/
@@ -57,7 +58,8 @@
* @param conf - Configuration.
* @return A instance of {@link ContainerCache}.
*/
- public synchronized static ContainerCache getInstance(Configuration conf) {
+ public synchronized static ContainerCache getInstance(
+ ConfigurationSource conf) {
if (cache == null) {
int cacheSize = conf.getInt(OzoneConfigKeys.OZONE_CONTAINER_CACHE_SIZE,
OzoneConfigKeys.OZONE_CONTAINER_CACHE_DEFAULT);
@@ -111,7 +113,7 @@
* @return ReferenceCountedDB.
*/
public ReferenceCountedDB getDB(long containerID, String containerDBType,
- String containerDBPath, Configuration conf)
+ String containerDBPath, ConfigurationSource conf)
throws IOException {
Preconditions.checkState(containerID >= 0,
"Container ID cannot be negative.");
@@ -160,4 +162,19 @@
lock.unlock();
}
}
+
+ /**
+ * Add a DB handler into cache.
+ *
+ * @param containerDBPath - DB path of the container.
+ * @param db - DB handler
+ */
+ public void addDB(String containerDBPath, ReferenceCountedDB db) {
+ lock.lock();
+ try {
+ this.putIfAbsent(containerDBPath, db);
+ } finally {
+ lock.unlock();
+ }
+ }
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ReferenceCountedDB.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ReferenceCountedDB.java
index fb143a4..5ab91a8 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ReferenceCountedDB.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ReferenceCountedDB.java
@@ -20,7 +20,7 @@
import com.google.common.base.Preconditions;
-import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.hadoop.hdds.utils.MetadataStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
index a9bfc5e..c3a5a41 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
@@ -19,11 +19,16 @@
package org.apache.hadoop.ozone.container.common.volume;
import javax.annotation.Nullable;
+import java.io.File;
+import java.io.IOException;
+import java.util.Objects;
+import java.util.Properties;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.fs.SpaceUsageCheckFactory;
import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.fs.SpaceUsageCheckFactory;
import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
import org.apache.hadoop.hdfs.server.datanode.checker.Checkable;
import org.apache.hadoop.hdfs.server.datanode.checker.VolumeCheckResult;
@@ -31,21 +36,15 @@
import org.apache.hadoop.ozone.container.common.DataNodeLayoutVersion;
import org.apache.hadoop.ozone.container.common.helpers.DatanodeVersionFile;
import org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil;
-
import org.apache.hadoop.util.DiskChecker;
import org.apache.hadoop.util.Time;
+
+import com.google.common.base.Preconditions;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.io.IOException;
-import java.util.Objects;
-import java.util.Properties;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicLong;
-
/**
* HddsVolume represents volume in a datanode. {@link MutableVolumeSet}
* maintains a list of HddsVolumes, one for each volume in the Datanode.
@@ -109,7 +108,7 @@
*/
public static class Builder {
private final String volumeRootStr;
- private Configuration conf;
+ private ConfigurationSource conf;
private StorageType storageType;
private String datanodeUuid;
@@ -121,7 +120,7 @@
this.volumeRootStr = rootDirStr;
}
- public Builder conf(Configuration config) {
+ public Builder conf(ConfigurationSource config) {
this.conf = config;
return this;
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolumeChecker.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolumeChecker.java
index ce5d16b..9240a85 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolumeChecker.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolumeChecker.java
@@ -32,7 +32,7 @@
import java.util.concurrent.atomic.AtomicLong;
import com.google.common.util.concurrent.MoreExecutors;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.DFSConfigKeysLegacy;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.checker.VolumeCheckResult;
@@ -91,7 +91,7 @@
* @param conf Configuration object.
* @param timer {@link Timer} object used for throttling checks.
*/
- public HddsVolumeChecker(Configuration conf, Timer timer)
+ public HddsVolumeChecker(ConfigurationSource conf, Timer timer)
throws DiskErrorException {
maxAllowedTimeForCheckMs = conf.getTimeDuration(
DFSConfigKeysLegacy.DFS_DATANODE_DISK_CHECK_TIMEOUT_KEY,
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java
index 101e680..bc61811 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java
@@ -33,9 +33,9 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.fs.SpaceUsageCheckFactory;
import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.fs.SpaceUsageCheckFactory;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
@@ -65,7 +65,7 @@
private static final Logger LOG =
LoggerFactory.getLogger(MutableVolumeSet.class);
- private Configuration conf;
+ private ConfigurationSource conf;
/**
* Maintains a map of all active volumes in the DataNode.
@@ -106,12 +106,13 @@
private final HddsVolumeChecker volumeChecker;
private Runnable failedVolumeListener;
- public MutableVolumeSet(String dnUuid, Configuration conf)
+ public MutableVolumeSet(String dnUuid, ConfigurationSource conf)
throws IOException {
this(dnUuid, null, conf);
}
- public MutableVolumeSet(String dnUuid, String clusterID, Configuration conf)
+ public MutableVolumeSet(String dnUuid, String clusterID,
+ ConfigurationSource conf)
throws IOException {
this.datanodeUuid = dnUuid;
this.clusterID = clusterID;
@@ -144,7 +145,7 @@
}
@VisibleForTesting
- HddsVolumeChecker getVolumeChecker(Configuration configuration)
+ HddsVolumeChecker getVolumeChecker(ConfigurationSource configuration)
throws DiskChecker.DiskErrorException {
return new HddsVolumeChecker(configuration, new Timer());
}
@@ -162,15 +163,7 @@
failedVolumeMap = new ConcurrentHashMap<>();
volumeStateMap = new EnumMap<>(StorageType.class);
- Collection<String> rawLocations = conf.getTrimmedStringCollection(
- HDDS_DATANODE_DIR_KEY);
- if (rawLocations.isEmpty()) {
- rawLocations = conf.getTrimmedStringCollection(DFS_DATANODE_DATA_DIR_KEY);
- }
- if (rawLocations.isEmpty()) {
- throw new IllegalArgumentException("No location configured in either "
- + HDDS_DATANODE_DIR_KEY + " or " + DFS_DATANODE_DATA_DIR_KEY);
- }
+ Collection<String> rawLocations = getDatanodeStorageDirs(conf);
for (StorageType storageType : StorageType.values()) {
volumeStateMap.put(storageType, new ArrayList<>());
@@ -219,6 +212,20 @@
SHUTDOWN_HOOK_PRIORITY);
}
+ public static Collection<String> getDatanodeStorageDirs(
+ ConfigurationSource conf) {
+ Collection<String> rawLocations = conf.getTrimmedStringCollection(
+ HDDS_DATANODE_DIR_KEY);
+ if (rawLocations.isEmpty()) {
+ rawLocations = conf.getTrimmedStringCollection(DFS_DATANODE_DATA_DIR_KEY);
+ }
+ if (rawLocations.isEmpty()) {
+ throw new IllegalArgumentException("No location configured in either "
+ + HDDS_DATANODE_DIR_KEY + " or " + DFS_DATANODE_DATA_DIR_KEY);
+ }
+ return rawLocations;
+ }
+
/**
* Run a synchronous parallel check of all HDDS volumes, removing
* failed volumes.
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
index 215d1e5..e0669c7 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
@@ -18,17 +18,18 @@
package org.apache.hadoop.ozone.container.common.volume;
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.fs.SpaceUsageCheckFactory;
-import org.apache.hadoop.hdds.fs.SpaceUsageCheckParams;
-import org.apache.hadoop.fs.StorageType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.File;
import java.io.IOException;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.fs.SpaceUsageCheckFactory;
+import org.apache.hadoop.hdds.fs.SpaceUsageCheckParams;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* Stores information about a disk/volume.
*/
@@ -51,13 +52,13 @@
* Builder for VolumeInfo.
*/
public static class Builder {
- private final Configuration conf;
+ private final ConfigurationSource conf;
private final String rootDir;
private SpaceUsageCheckFactory usageCheckFactory;
private StorageType storageType;
private long configuredCapacity;
- public Builder(String root, Configuration config) {
+ public Builder(String root, ConfigurationSource config) {
this.rootDir = root;
this.conf = config;
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
index 6317e63..9892cb5 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
@@ -28,18 +28,14 @@
import java.util.Map;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerDataProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerType;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
-import org.apache.hadoop.hdds.scm.container.common.helpers
- .StorageContainerException;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerType;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.hdfs.util.Canceler;
import org.apache.hadoop.hdfs.util.DataTransferThrottler;
import org.apache.hadoop.io.nativeio.NativeIO;
@@ -50,35 +46,25 @@
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerPacker;
import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy;
+import org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
-import org.apache.hadoop.ozone.container.keyvalue.helpers
- .KeyValueContainerLocationUtil;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerLocationUtil;
import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil;
import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
import com.google.common.base.Preconditions;
import org.apache.commons.io.FileUtils;
-import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .Result.CONTAINER_ALREADY_EXISTS;
-import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .Result.CONTAINER_FILES_CREATE_ERROR;
-import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .Result.CONTAINER_INTERNAL_ERROR;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CONTAINER_ALREADY_EXISTS;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CONTAINER_FILES_CREATE_ERROR;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CONTAINER_INTERNAL_ERROR;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CONTAINER_NOT_OPEN;
-import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .Result.DISK_OUT_OF_SPACE;
-import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .Result.ERROR_IN_COMPACT_DB;
-import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .Result.ERROR_IN_DB_SYNC;
-import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .Result.INVALID_CONTAINER_STATE;
-import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .Result.UNSUPPORTED_REQUEST;
-
-import org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.DISK_OUT_OF_SPACE;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.ERROR_IN_COMPACT_DB;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.ERROR_IN_DB_SYNC;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.INVALID_CONTAINER_STATE;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.UNSUPPORTED_REQUEST;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -96,9 +82,10 @@
private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
private final KeyValueContainerData containerData;
- private Configuration config;
+ private ConfigurationSource config;
- public KeyValueContainer(KeyValueContainerData containerData, Configuration
+ public KeyValueContainer(KeyValueContainerData containerData,
+ ConfigurationSource
ozoneConfig) {
Preconditions.checkNotNull(containerData, "KeyValueContainerData cannot " +
"be null");
@@ -505,7 +492,6 @@
containerData.setState(originalContainerData.getState());
containerData
.setContainerDBType(originalContainerData.getContainerDBType());
- containerData.setBytesUsed(originalContainerData.getBytesUsed());
//rewriting the yaml file with new checksum calculation.
update(originalContainerData.getMetadata(), true);
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerCheck.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerCheck.java
index 1e53daa..95795e6 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerCheck.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerCheck.java
@@ -20,7 +20,7 @@
import com.google.common.base.Preconditions;
import com.google.common.primitives.Longs;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdfs.util.Canceler;
import org.apache.hadoop.hdfs.util.DataTransferThrottler;
@@ -62,11 +62,11 @@
private long containerID;
private KeyValueContainerData onDiskContainerData; //loaded from fs/disk
- private Configuration checkConfig;
+ private ConfigurationSource checkConfig;
private String metadataPath;
- public KeyValueContainerCheck(String metadataPath, Configuration conf,
+ public KeyValueContainerCheck(String metadataPath, ConfigurationSource conf,
long containerID) {
Preconditions.checkArgument(metadataPath != null);
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
index 1e373de..373b322 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
@@ -20,13 +20,18 @@
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+
+import java.io.IOException;
import java.util.Collections;
+import com.google.common.primitives.Longs;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerDataProto;
+import org.apache.hadoop.hdds.utils.BatchOperation;
import org.apache.hadoop.ozone.container.common.impl.ChunkLayOutVersion;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
+import org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB;
import org.yaml.snakeyaml.nodes.Tag;
@@ -36,9 +41,12 @@
import java.util.concurrent.atomic.AtomicInteger;
import static java.lang.Math.max;
+import static org.apache.hadoop.ozone.OzoneConsts.DB_BLOCK_COUNT_KEY;
import static org.apache.hadoop.ozone.OzoneConsts.CHUNKS_PATH;
+import static org.apache.hadoop.ozone.OzoneConsts.DB_CONTAINER_BYTES_USED_KEY;
import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_TYPE;
import static org.apache.hadoop.ozone.OzoneConsts.METADATA_PATH;
+import static org.apache.hadoop.ozone.OzoneConsts.DB_PENDING_DELETE_BLOCK_COUNT_KEY;
/**
* This class represents the KeyValueContainer metadata, which is the
@@ -141,9 +149,6 @@
* @return Path to base dir
*/
public String getContainerPath() {
- if (metadataPath == null) {
- return null;
- }
return new File(metadataPath).getParent();
}
@@ -226,7 +231,7 @@
public ContainerDataProto getProtoBufMessage() {
ContainerDataProto.Builder builder = ContainerDataProto.newBuilder();
builder.setContainerID(this.getContainerID());
- builder.setContainerPath(this.getMetadataPath());
+ builder.setContainerPath(this.getContainerPath());
builder.setState(this.getState());
for (Map.Entry<String, String> entry : getMetadata().entrySet()) {
@@ -251,4 +256,25 @@
return Collections.unmodifiableList(KV_YAML_FIELDS);
}
+ /**
+ * Update DB counters related to block metadata.
+ * @param db - Reference to container DB.
+ * @param batchOperation - Batch Operation to batch DB operations.
+ * @param deletedBlockCount - Number of blocks deleted.
+ * @throws IOException
+ */
+ public void updateAndCommitDBCounters(
+ ReferenceCountedDB db, BatchOperation batchOperation,
+ int deletedBlockCount) throws IOException {
+ // Set Bytes used and block count key.
+ batchOperation.put(DB_CONTAINER_BYTES_USED_KEY,
+ Longs.toByteArray(getBytesUsed()));
+ batchOperation.put(DB_BLOCK_COUNT_KEY, Longs.toByteArray(
+ getKeyCount() - deletedBlockCount));
+ batchOperation.put(DB_PENDING_DELETE_BLOCK_COUNT_KEY, Longs.toByteArray(
+ getNumPendingDeletionBlocks() - deletedBlockCount));
+ db.getStore().writeBatch(batchOperation);
+ }
+
+
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index a329bdb..26e98c2 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -30,30 +30,23 @@
import java.util.function.Consumer;
import java.util.function.Function;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerDataProto.State;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerCommandRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerCommandResponseProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerType;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .GetSmallFileRequestProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerType;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.GetSmallFileRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyValue;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .PutSmallFileRequestProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.PutSmallFileRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.WriteChunkRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
import org.apache.hadoop.hdds.scm.ByteStringConversion;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.container.common.helpers
- .StorageContainerException;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.ozone.common.ChunkBuffer;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
@@ -65,28 +58,28 @@
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.interfaces.Handler;
import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy;
-import org.apache.hadoop.ozone.container.common.transport.server.ratis
- .DispatcherContext;
-import org.apache.hadoop.ozone.container.common.transport.server.ratis
- .DispatcherContext.WriteChunkStage;
+import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext;
+import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext.WriteChunkStage;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
-import org.apache.hadoop.ozone.container.common.volume
- .RoundRobinVolumeChoosingPolicy;
+import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
-import org.apache.hadoop.ozone.container.keyvalue.impl.ChunkManagerFactory;
import org.apache.hadoop.ozone.container.keyvalue.impl.BlockManagerImpl;
-import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager;
+import org.apache.hadoop.ozone.container.keyvalue.impl.ChunkManagerFactory;
import org.apache.hadoop.ozone.container.keyvalue.interfaces.BlockManager;
+import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager;
import org.apache.hadoop.util.AutoCloseableLock;
-import org.apache.hadoop.util.ReflectionUtils;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
-import static org.apache.hadoop.hdds.HddsConfigKeys
- .HDDS_DATANODE_VOLUME_CHOOSING_POLICY;
-import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.
- Result.*;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_VOLUME_CHOOSING_POLICY;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CLOSED_CONTAINER_IO;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CONTAINER_INTERNAL_ERROR;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CONTAINER_UNHEALTHY;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.DELETE_ON_OPEN_CONTAINER;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.GET_SMALL_FILE_ERROR;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.INVALID_CONTAINER_STATE;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.IO_EXCEPTION;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.PUT_SMALL_FILE_ERROR;
import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.getBlockDataResponse;
import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.getBlockLengthResponse;
import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.getBlockResponseSuccess;
@@ -98,7 +91,7 @@
import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.malformedRequest;
import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.putBlockResponseSuccess;
import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.unsupportedRequest;
-
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -120,16 +113,20 @@
// A lock that is held during container creation.
private final AutoCloseableLock containerCreationLock;
- public KeyValueHandler(Configuration config, String datanodeId,
+ public KeyValueHandler(ConfigurationSource config, String datanodeId,
ContainerSet contSet, VolumeSet volSet, ContainerMetrics metrics,
Consumer<ContainerReplicaProto> icrSender) {
super(config, datanodeId, contSet, volSet, metrics, icrSender);
containerType = ContainerType.KeyValueContainer;
blockManager = new BlockManagerImpl(config);
chunkManager = ChunkManagerFactory.createChunkManager(config);
- volumeChoosingPolicy = ReflectionUtils.newInstance(conf.getClass(
- HDDS_DATANODE_VOLUME_CHOOSING_POLICY, RoundRobinVolumeChoosingPolicy
- .class, VolumeChoosingPolicy.class), conf);
+ try {
+ volumeChoosingPolicy = conf.getClass(
+ HDDS_DATANODE_VOLUME_CHOOSING_POLICY, RoundRobinVolumeChoosingPolicy
+ .class, VolumeChoosingPolicy.class).newInstance();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
maxContainerSize = (long)config.getStorageSize(
ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
@@ -154,45 +151,55 @@
ContainerCommandRequestProto request, Container container,
DispatcherContext dispatcherContext) {
+ return KeyValueHandler
+ .dispatchRequest(this, request, (KeyValueContainer) container,
+ dispatcherContext);
+ }
+
+ @VisibleForTesting
+ static ContainerCommandResponseProto dispatchRequest(KeyValueHandler handler,
+ ContainerCommandRequestProto request, KeyValueContainer kvContainer,
+ DispatcherContext dispatcherContext) {
Type cmdType = request.getCmdType();
- KeyValueContainer kvContainer = (KeyValueContainer) container;
+
switch(cmdType) {
case CreateContainer:
- return handleCreateContainer(request, kvContainer);
+ return handler.handleCreateContainer(request, kvContainer);
case ReadContainer:
- return handleReadContainer(request, kvContainer);
+ return handler.handleReadContainer(request, kvContainer);
case UpdateContainer:
- return handleUpdateContainer(request, kvContainer);
+ return handler.handleUpdateContainer(request, kvContainer);
case DeleteContainer:
- return handleDeleteContainer(request, kvContainer);
+ return handler.handleDeleteContainer(request, kvContainer);
case ListContainer:
- return handleUnsupportedOp(request);
+ return handler.handleUnsupportedOp(request);
case CloseContainer:
- return handleCloseContainer(request, kvContainer);
+ return handler.handleCloseContainer(request, kvContainer);
case PutBlock:
- return handlePutBlock(request, kvContainer, dispatcherContext);
+ return handler.handlePutBlock(request, kvContainer, dispatcherContext);
case GetBlock:
- return handleGetBlock(request, kvContainer);
+ return handler.handleGetBlock(request, kvContainer);
case DeleteBlock:
- return handleDeleteBlock(request, kvContainer);
+ return handler.handleDeleteBlock(request, kvContainer);
case ListBlock:
- return handleUnsupportedOp(request);
+ return handler.handleUnsupportedOp(request);
case ReadChunk:
- return handleReadChunk(request, kvContainer, dispatcherContext);
+ return handler.handleReadChunk(request, kvContainer, dispatcherContext);
case DeleteChunk:
- return handleDeleteChunk(request, kvContainer);
+ return handler.handleDeleteChunk(request, kvContainer);
case WriteChunk:
- return handleWriteChunk(request, kvContainer, dispatcherContext);
+ return handler.handleWriteChunk(request, kvContainer, dispatcherContext);
case ListChunk:
- return handleUnsupportedOp(request);
+ return handler.handleUnsupportedOp(request);
case CompactChunk:
- return handleUnsupportedOp(request);
+ return handler.handleUnsupportedOp(request);
case PutSmallFile:
- return handlePutSmallFile(request, kvContainer, dispatcherContext);
+ return handler
+ .handlePutSmallFile(request, kvContainer, dispatcherContext);
case GetSmallFile:
- return handleGetSmallFile(request, kvContainer);
+ return handler.handleGetSmallFile(request, kvContainer);
case GetCommittedBlockLength:
- return handleGetCommittedBlockLength(request, kvContainer);
+ return handler.handleGetCommittedBlockLength(request, kvContainer);
default:
return null;
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
index 35e0b0c..99ead01 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
@@ -18,21 +18,19 @@
package org.apache.hadoop.ozone.container.keyvalue.helpers;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
+import java.io.IOException;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
-import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.common.utils.ContainerCache;
import org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
-import java.io.IOException;
-
-import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .Result.NO_SUCH_BLOCK;
-import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .Result.UNABLE_TO_READ_METADATA_DB;
+import com.google.common.base.Preconditions;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.NO_SUCH_BLOCK;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.UNABLE_TO_READ_METADATA_DB;
/**
* Utils functions to help block functions.
@@ -55,7 +53,7 @@
* @throws StorageContainerException
*/
public static ReferenceCountedDB getDB(KeyValueContainerData containerData,
- Configuration conf) throws
+ ConfigurationSource conf) throws
StorageContainerException {
Preconditions.checkNotNull(containerData);
ContainerCache cache = ContainerCache.getInstance(conf);
@@ -78,8 +76,8 @@
* @param container - Container data.
* @param conf - Configuration.
*/
- public static void removeDB(KeyValueContainerData container, Configuration
- conf) {
+ public static void removeDB(KeyValueContainerData container,
+ ConfigurationSource conf) {
Preconditions.checkNotNull(container);
ContainerCache cache = ContainerCache.getInstance(conf);
Preconditions.checkNotNull(cache);
@@ -96,6 +94,20 @@
}
/**
+ * Add a DB handler into cache.
+ *
+ * @param db - DB handler.
+ * @param containerDBPath - DB path of the container.
+ * @param conf configuration.
+ */
+ public static void addDB(ReferenceCountedDB db, String containerDBPath,
+ ConfigurationSource conf) {
+ ContainerCache cache = ContainerCache.getInstance(conf);
+ Preconditions.checkNotNull(cache);
+ cache.addDB(containerDBPath, db);
+ }
+
+ /**
* Parses the {@link BlockData} from a bytes array.
*
* @param bytes Block data in bytes.
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java
index 501680a..2141bed 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java
@@ -22,16 +22,18 @@
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
-import java.util.Map;
+import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.hadoop.ozone.container.common.helpers.BlockData;
-import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.utils.MetadataKeyFilters;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
+import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueBlockIterator;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.hdds.utils.MetadataStore;
import org.apache.hadoop.hdds.utils.MetadataStoreBuilder;
@@ -41,6 +43,12 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.ozone.OzoneConsts.DB_BLOCK_COMMIT_SEQUENCE_ID_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.DB_BLOCK_COUNT_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.DB_CONTAINER_BYTES_USED_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.DB_CONTAINER_DELETE_TRANSACTION_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.DB_PENDING_DELETE_BLOCK_COUNT_KEY;
+
/**
* Class which defines utility methods for KeyValueContainer.
*/
@@ -63,7 +71,7 @@
* @throws IOException
*/
public static void createContainerMetaData(File containerMetaDataPath, File
- chunksPath, File dbFile, Configuration conf) throws IOException {
+ chunksPath, File dbFile, ConfigurationSource conf) throws IOException {
Preconditions.checkNotNull(containerMetaDataPath);
Preconditions.checkNotNull(conf);
@@ -73,14 +81,6 @@
throw new IOException("Unable to create directory for metadata storage." +
" Path: " + containerMetaDataPath);
}
- MetadataStore store = MetadataStoreBuilder.newBuilder().setConf(conf)
- .setCreateIfMissing(true).setDbFile(dbFile).build();
-
- // we close since the SCM pre-creates containers.
- // we will open and put Db handle into a cache when keys are being created
- // in a container.
-
- store.close();
if (!chunksPath.mkdirs()) {
LOG.error("Unable to create chunks directory Container {}",
@@ -91,6 +91,13 @@
throw new IOException("Unable to create directory for data storage." +
" Path: " + chunksPath);
}
+
+ MetadataStore store = MetadataStoreBuilder.newBuilder().setConf(conf)
+ .setCreateIfMissing(true).setDbFile(dbFile).build();
+ ReferenceCountedDB db =
+ new ReferenceCountedDB(store, dbFile.getAbsolutePath());
+ //add db handler into cache
+ BlockUtils.addDB(db, dbFile.getAbsolutePath(), conf);
}
/**
@@ -106,7 +113,7 @@
* @throws IOException
*/
public static void removeContainer(KeyValueContainerData containerData,
- Configuration conf)
+ ConfigurationSource conf)
throws IOException {
Preconditions.checkNotNull(containerData);
File containerMetaDataPath = new File(containerData
@@ -127,13 +134,15 @@
}
/**
- * Parse KeyValueContainerData and verify checksum.
+ * Parse KeyValueContainerData and verify checksum. Set block related
+ * metadata like block commit sequence id, block count, bytes used and
+ * pending delete block count and delete transaction id.
* @param kvContainerData
* @param config
* @throws IOException
*/
public static void parseKVContainerData(KeyValueContainerData kvContainerData,
- Configuration config) throws IOException {
+ ConfigurationSource config) throws IOException {
long containerID = kvContainerData.getContainerID();
File metadataPath = new File(kvContainerData.getMetadataPath());
@@ -151,29 +160,109 @@
}
kvContainerData.setDbFile(dbFile);
- try(ReferenceCountedDB metadata =
- BlockUtils.getDB(kvContainerData, config)) {
- long bytesUsed = 0;
- List<Map.Entry<byte[], byte[]>> liveKeys = metadata.getStore()
- .getRangeKVs(null, Integer.MAX_VALUE,
- MetadataKeyFilters.getNormalKeyFilter());
- bytesUsed = liveKeys.parallelStream().mapToLong(e-> {
- BlockData blockData;
- try {
- blockData = BlockUtils.getBlockData(e.getValue());
- return blockData.getSize();
- } catch (IOException ex) {
- return 0L;
- }
- }).sum();
- kvContainerData.setBytesUsed(bytesUsed);
- kvContainerData.setKeyCount(liveKeys.size());
- byte[] bcsId = metadata.getStore().get(DFSUtil.string2Bytes(
- OzoneConsts.BLOCK_COMMIT_SEQUENCE_ID_PREFIX));
- if (bcsId != null) {
- kvContainerData.updateBlockCommitSequenceId(Longs.fromByteArray(bcsId));
+ boolean isBlockMetadataSet = false;
+
+ try(ReferenceCountedDB containerDB = BlockUtils.getDB(kvContainerData,
+ config)) {
+
+ // Set pending deleted block count.
+ byte[] pendingDeleteBlockCount =
+ containerDB.getStore().get(DB_PENDING_DELETE_BLOCK_COUNT_KEY);
+ if (pendingDeleteBlockCount != null) {
+ kvContainerData.incrPendingDeletionBlocks(
+ Ints.fromByteArray(pendingDeleteBlockCount));
+ } else {
+ // Set pending deleted block count.
+ MetadataKeyFilters.KeyPrefixFilter filter =
+ new MetadataKeyFilters.KeyPrefixFilter()
+ .addFilter(OzoneConsts.DELETING_KEY_PREFIX);
+ int numPendingDeletionBlocks =
+ containerDB.getStore().getSequentialRangeKVs(null,
+ Integer.MAX_VALUE, filter)
+ .size();
+ kvContainerData.incrPendingDeletionBlocks(numPendingDeletionBlocks);
}
+
+ // Set delete transaction id.
+ byte[] delTxnId =
+ containerDB.getStore().get(DB_CONTAINER_DELETE_TRANSACTION_KEY);
+ if (delTxnId != null) {
+ kvContainerData
+ .updateDeleteTransactionId(Longs.fromByteArray(delTxnId));
+ }
+
+ // Set BlockCommitSequenceId.
+ byte[] bcsId = containerDB.getStore().get(
+ DB_BLOCK_COMMIT_SEQUENCE_ID_KEY);
+ if (bcsId != null) {
+ kvContainerData
+ .updateBlockCommitSequenceId(Longs.fromByteArray(bcsId));
+ }
+
+ // Set bytes used.
+ // commitSpace for Open Containers relies on usedBytes
+ byte[] bytesUsed =
+ containerDB.getStore().get(DB_CONTAINER_BYTES_USED_KEY);
+ if (bytesUsed != null) {
+ isBlockMetadataSet = true;
+ kvContainerData.setBytesUsed(Longs.fromByteArray(bytesUsed));
+ }
+
+ // Set block count.
+ byte[] blockCount = containerDB.getStore().get(DB_BLOCK_COUNT_KEY);
+ if (blockCount != null) {
+ isBlockMetadataSet = true;
+ kvContainerData.setKeyCount(Longs.fromByteArray(blockCount));
+ }
+ }
+
+ if (!isBlockMetadataSet) {
+ initializeUsedBytesAndBlockCount(kvContainerData);
+ }
+ }
+
+
+ /**
+ * Initialize bytes used and block count.
+ * @param kvContainerData
+ * @throws IOException
+ */
+ private static void initializeUsedBytesAndBlockCount(
+ KeyValueContainerData kvContainerData) throws IOException {
+
+ long blockCount = 0;
+ try (KeyValueBlockIterator blockIter = new KeyValueBlockIterator(
+ kvContainerData.getContainerID(),
+ new File(kvContainerData.getContainerPath()))) {
+ long usedBytes = 0;
+
+
+ boolean success = true;
+ while (success) {
+ try {
+ if (blockIter.hasNext()) {
+ BlockData block = blockIter.nextBlock();
+ long blockLen = 0;
+
+ List< ContainerProtos.ChunkInfo > chunkInfoList = block.getChunks();
+ for (ContainerProtos.ChunkInfo chunk : chunkInfoList) {
+ ChunkInfo info = ChunkInfo.getFromProtoBuf(chunk);
+ blockLen += info.getLen();
+ }
+
+ usedBytes += blockLen;
+ blockCount++;
+ } else {
+ success = false;
+ }
+ } catch (IOException ex) {
+ LOG.error("Failed to parse block data for Container {}",
+ kvContainerData.getContainerID());
+ }
+ }
+ kvContainerData.setBytesUsed(usedBytes);
+ kvContainerData.setKeyCount(blockCount);
}
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
index 58a4e8b..940383e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
@@ -20,13 +20,11 @@
import com.google.common.base.Preconditions;
import com.google.common.primitives.Longs;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
@@ -47,6 +45,10 @@
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.NO_SUCH_BLOCK;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.UNKNOWN_BCSID;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.BCSID_MISMATCH;
+import static org.apache.hadoop.ozone.OzoneConsts.DB_BLOCK_COMMIT_SEQUENCE_ID_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.DB_BLOCK_COUNT_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.DB_CONTAINER_BYTES_USED_KEY;
+
/**
* This class is for performing block related operations on the KeyValue
* Container.
@@ -54,10 +56,9 @@
public class BlockManagerImpl implements BlockManager {
static final Logger LOG = LoggerFactory.getLogger(BlockManagerImpl.class);
- private static byte[] blockCommitSequenceIdKey =
- DFSUtil.string2Bytes(OzoneConsts.BLOCK_COMMIT_SEQUENCE_ID_PREFIX);
- private Configuration config;
+
+ private ConfigurationSource config;
private static final String DB_NULL_ERR_MSG = "DB cannot be null here";
private static final String NO_SUCH_BLOCK_ERR_MSG =
@@ -68,7 +69,7 @@
*
* @param conf - Ozone configuration
*/
- public BlockManagerImpl(Configuration conf) {
+ public BlockManagerImpl(ConfigurationSource conf) {
Preconditions.checkNotNull(conf, "Config cannot be null");
this.config = conf;
}
@@ -116,11 +117,25 @@
BatchOperation batch = new BatchOperation();
batch.put(Longs.toByteArray(data.getLocalID()),
data.getProtoBufMessage().toByteArray());
- batch.put(blockCommitSequenceIdKey,
- Longs.toByteArray(bcsId));
+ batch.put(DB_BLOCK_COMMIT_SEQUENCE_ID_KEY, Longs.toByteArray(bcsId));
+
+ // Set Bytes used, this bytes used will be updated for every write and
+ // only get committed for every put block. In this way, when datanode
+ // is up, for computation of disk space by container only committed
+ // block length is used, And also on restart the blocks committed to DB
+ // is only used to compute the bytes used. This is done to keep the
+ // current behavior and avoid DB write during write chunk operation.
+ batch.put(DB_CONTAINER_BYTES_USED_KEY,
+ Longs.toByteArray(container.getContainerData().getBytesUsed()));
+
+ // Set Block Count for a container.
+ batch.put(DB_BLOCK_COUNT_KEY,
+ Longs.toByteArray(container.getContainerData().getKeyCount() + 1));
+
db.getStore().writeBatch(batch);
+
container.updateBlockCommitSequenceId(bcsId);
- // Increment keycount here
+ // Increment block count finally here for in-memory.
container.getContainerData().incrKeyCount();
if (LOG.isDebugEnabled()) {
LOG.debug(
@@ -224,11 +239,21 @@
// are not atomic. Leaving it here since the impact is refusing
// to delete a Block which might have just gotten inserted after
// the get check.
- byte[] kKey = Longs.toByteArray(blockID.getLocalID());
+ byte[] blockKey = Longs.toByteArray(blockID.getLocalID());
getBlockByID(db, blockID);
- db.getStore().delete(kKey);
- // Decrement blockcount here
+
+ // Update DB to delete block and set block count and bytes used.
+ BatchOperation batch = new BatchOperation();
+ batch.delete(blockKey);
+ // Update DB to delete block and set block count.
+ // No need to set bytes used here, as bytes used is taken care during
+ // delete chunk.
+ batch.put(DB_BLOCK_COUNT_KEY,
+ Longs.toByteArray(container.getContainerData().getKeyCount() - 1));
+ db.getStore().writeBatch(batch);
+
+ // Decrement block count here
container.getContainerData().decrKeyCount();
}
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerFactory.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerFactory.java
index c2d81b6..6b3f58d 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerFactory.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerFactory.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.ozone.container.keyvalue.impl;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager;
import org.slf4j.Logger;
@@ -39,7 +39,7 @@
private ChunkManagerFactory() {
}
- public static ChunkManager createChunkManager(Configuration conf) {
+ public static ChunkManager createChunkManager(ConfigurationSource conf) {
boolean sync =
conf.getBoolean(OzoneConfigKeys.DFS_CONTAINER_CHUNK_WRITE_SYNC_KEY,
OzoneConfigKeys.DFS_CONTAINER_CHUNK_WRITE_SYNC_DEFAULT);
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
index 798629e..375263c 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
@@ -18,39 +18,6 @@
package org.apache.hadoop.ozone.container.keyvalue.statemachine.background;
-import com.google.common.collect.Lists;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
-import org.apache.hadoop.ozone.container.common.helpers.BlockData;
-import org.apache.hadoop.ozone.container.common.impl.ContainerData;
-import org.apache.hadoop.ozone.container.common.impl.TopNOrderedContainerDeletionChoosingPolicy;
-import org.apache.hadoop.ozone.container.common.interfaces.Container;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDeletionChoosingPolicy;
-import org.apache.hadoop.ozone.container.common.interfaces.Handler;
-import org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis;
-import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
-import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
-import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.ratis.thirdparty.com.google.protobuf
- .InvalidProtocolBufferException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.scm.container.common.helpers
- .StorageContainerException;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.hdds.utils.BackgroundService;
-import org.apache.hadoop.hdds.utils.BackgroundTask;
-import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
-import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
-import org.apache.hadoop.hdds.utils.BatchOperation;
-import org.apache.hadoop.hdds.utils.MetadataKeyFilters.KeyPrefixFilter;
-import org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.File;
import java.io.IOException;
import java.util.LinkedList;
@@ -61,14 +28,40 @@
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER_DEFAULT;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BatchOperation;
+import org.apache.hadoop.hdds.utils.MetadataKeyFilters.KeyPrefixFilter;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
+import org.apache.hadoop.ozone.container.common.impl.ContainerData;
+import org.apache.hadoop.ozone.container.common.impl.TopNOrderedContainerDeletionChoosingPolicy;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.common.interfaces.ContainerDeletionChoosingPolicy;
+import org.apache.hadoop.ozone.container.common.interfaces.Handler;
+import org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis;
+import org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
+import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+import org.apache.hadoop.util.Time;
+
+import com.google.common.collect.Lists;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER_DEFAULT;
+import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A per-datanode container block deleting service takes in charge
@@ -82,7 +75,7 @@
private OzoneContainer ozoneContainer;
private ContainerDeletionChoosingPolicy containerDeletionPolicy;
- private final Configuration conf;
+ private final ConfigurationSource conf;
// Throttle number of blocks to delete per task,
// set to 1 for testing
@@ -98,14 +91,18 @@
public BlockDeletingService(OzoneContainer ozoneContainer,
long serviceInterval, long serviceTimeout, TimeUnit timeUnit,
- Configuration conf) {
+ ConfigurationSource conf) {
super("BlockDeletingService", serviceInterval, timeUnit,
BLOCK_DELETING_SERVICE_CORE_POOL_SIZE, serviceTimeout);
this.ozoneContainer = ozoneContainer;
- containerDeletionPolicy = ReflectionUtils.newInstance(conf.getClass(
- ScmConfigKeys.OZONE_SCM_KEY_VALUE_CONTAINER_DELETION_CHOOSING_POLICY,
- TopNOrderedContainerDeletionChoosingPolicy.class,
- ContainerDeletionChoosingPolicy.class), conf);
+ try {
+ containerDeletionPolicy = conf.getClass(
+ ScmConfigKeys.OZONE_SCM_KEY_VALUE_CONTAINER_DELETION_CHOOSING_POLICY,
+ TopNOrderedContainerDeletionChoosingPolicy.class,
+ ContainerDeletionChoosingPolicy.class).newInstance();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
this.conf = conf;
this.blockLimitPerTask =
conf.getInt(OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER,
@@ -306,9 +303,15 @@
DFSUtil.string2Bytes(blockId));
batch.delete(DFSUtil.string2Bytes(entry));
});
- meta.getStore().writeBatch(batch);
- // update count of pending deletion blocks in in-memory container status
- containerData.decrPendingDeletionBlocks(succeedBlocks.size());
+
+
+ int deleteBlockCount = succeedBlocks.size();
+ containerData.updateAndCommitDBCounters(meta, batch, deleteBlockCount);
+
+ // update count of pending deletion blocks and block count in in-memory
+ // container status.
+ containerData.decrPendingDeletionBlocks(deleteBlockCount);
+ containerData.decrKeyCount(deleteBlockCount);
if (!succeedBlocks.isEmpty()) {
LOG.info("Container: {}, deleted blocks: {}, task elapsed time: {}ms",
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerMetadataScanner.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerMetadataScanner.java
index 50ea576..96efcf4 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerMetadataScanner.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerMetadataScanner.java
@@ -110,6 +110,10 @@
} catch (InterruptedException e) {
LOG.info("Background ContainerMetadataScanner interrupted." +
" Going to exit");
+ // Restore the interruption flag and the internal `stopping`
+ // variable to prevent the next iteration thus stopping the thread
+ interrupt();
+ this.stopping = true;
}
}
}
@@ -129,13 +133,19 @@
return metrics;
}
+ /**
+ * Shutdown the ContainerMetadataScanner thread.
+ */
+ // Ignore the sonar false positive on the InterruptedException issue
+ // as this a normal flow of a shutdown.
+ @SuppressWarnings("squid:S2142")
public synchronized void shutdown() {
this.stopping = true;
this.interrupt();
try {
this.join();
} catch (InterruptedException ex) {
- LOG.warn("Unexpected exception while stopping metadata scanner.", ex);
+ LOG.debug("Interrupted exception while stopping metadata scanner.", ex);
}
}
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java
index 297c1ca..1b9b3d6 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java
@@ -18,37 +18,27 @@
package org.apache.hadoop.ozone.container.ozoneimpl;
-import com.google.common.base.Preconditions;
-import com.google.common.primitives.Longs;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.scm.container.common.helpers
- .StorageContainerException;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.common.Storage;
-import org.apache.hadoop.ozone.container.common.helpers.BlockData;
-import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.hadoop.ozone.container.common.impl.ContainerData;
-import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
-import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
-import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
-import org.apache.hadoop.ozone.container.keyvalue.KeyValueBlockIterator;
-import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
-import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
-import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
-import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
-import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil;
-import org.apache.hadoop.hdds.utils.MetadataKeyFilters;
-import org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
-import java.util.List;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.ozone.common.Storage;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
+import org.apache.hadoop.ozone.container.common.impl.ContainerData;
+import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
+import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
+import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+
+import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Class used to read .container files from Volume and build container map.
@@ -81,12 +71,12 @@
ContainerReader.class);
private HddsVolume hddsVolume;
private final ContainerSet containerSet;
- private final OzoneConfiguration config;
+ private final ConfigurationSource config;
private final File hddsVolumeDir;
private final MutableVolumeSet volumeSet;
ContainerReader(MutableVolumeSet volSet, HddsVolume volume, ContainerSet cset,
- OzoneConfiguration conf) {
+ ConfigurationSource conf) {
Preconditions.checkNotNull(volume);
this.hddsVolume = volume;
this.hddsVolumeDir = hddsVolume.getHddsRootDir();
@@ -191,36 +181,8 @@
KeyValueContainerUtil.parseKVContainerData(kvContainerData, config);
KeyValueContainer kvContainer = new KeyValueContainer(
kvContainerData, config);
- try(ReferenceCountedDB containerDB = BlockUtils.getDB(kvContainerData,
- config)) {
- MetadataKeyFilters.KeyPrefixFilter filter =
- new MetadataKeyFilters.KeyPrefixFilter()
- .addFilter(OzoneConsts.DELETING_KEY_PREFIX);
- int numPendingDeletionBlocks =
- containerDB.getStore().getSequentialRangeKVs(null,
- Integer.MAX_VALUE, filter)
- .size();
- kvContainerData.incrPendingDeletionBlocks(numPendingDeletionBlocks);
- byte[] delTxnId = containerDB.getStore().get(
- DFSUtil.string2Bytes(OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX));
- if (delTxnId != null) {
- kvContainerData
- .updateDeleteTransactionId(Longs.fromByteArray(delTxnId));
- }
- // sets the BlockCommitSequenceId.
- byte[] bcsId = containerDB.getStore().get(DFSUtil.string2Bytes(
- OzoneConsts.BLOCK_COMMIT_SEQUENCE_ID_PREFIX));
- if (bcsId != null) {
- kvContainerData
- .updateBlockCommitSequenceId(Longs.fromByteArray(bcsId));
- }
- if (kvContainer.getContainerState()
- == ContainerProtos.ContainerDataProto.State.OPEN) {
- // commitSpace for Open Containers relies on usedBytes
- initializeUsedBytes(kvContainer);
- }
- containerSet.addContainer(kvContainer);
- }
+
+ containerSet.addContainer(kvContainer);
} else {
throw new StorageContainerException("Container File is corrupted. " +
"ContainerType is KeyValueContainer but cast to " +
@@ -234,28 +196,4 @@
ContainerProtos.Result.UNKNOWN_CONTAINER_TYPE);
}
}
-
- private void initializeUsedBytes(KeyValueContainer container)
- throws IOException {
- try (KeyValueBlockIterator blockIter = new KeyValueBlockIterator(
- container.getContainerData().getContainerID(),
- new File(container.getContainerData().getContainerPath()))) {
- long usedBytes = 0;
-
- while (blockIter.hasNext()) {
- BlockData block = blockIter.nextBlock();
- long blockLen = 0;
-
- List<ContainerProtos.ChunkInfo> chunkInfoList = block.getChunks();
- for (ContainerProtos.ChunkInfo chunk : chunkInfoList) {
- ChunkInfo info = ChunkInfo.getFromProtoBuf(chunk);
- blockLen += info.getLen();
- }
-
- usedBytes += blockLen;
- }
-
- container.getContainerData().setBytesUsed(usedBytes);
- }
- }
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index a713b85..bbbec25 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -26,7 +26,7 @@
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerType;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
@@ -72,7 +72,7 @@
private final HddsDispatcher hddsDispatcher;
private final Map<ContainerType, Handler> handlers;
- private final OzoneConfiguration config;
+ private final ConfigurationSource config;
private final MutableVolumeSet volumeSet;
private final ContainerSet containerSet;
private final XceiverServerSpi writeChannel;
@@ -90,7 +90,7 @@
* @throws DiskOutOfSpaceException
* @throws IOException
*/
- public OzoneContainer(DatanodeDetails datanodeDetails, OzoneConfiguration
+ public OzoneContainer(DatanodeDetails datanodeDetails, ConfigurationSource
conf, StateContext context, CertificateClient certClient)
throws IOException {
config = conf;
@@ -178,7 +178,8 @@
volumeThreads.get(i).join();
}
} catch (InterruptedException ex) {
- LOG.info("Volume Threads Interrupted exception", ex);
+ LOG.error("Volume Threads Interrupted exception", ex);
+ Thread.currentThread().interrupt();
}
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerStreamingOutput.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerStreamingOutput.java
deleted file mode 100644
index 4c65f80..0000000
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerStreamingOutput.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.replication;
-
-import javax.ws.rs.core.StreamingOutput;
-import java.io.IOException;
-import java.io.OutputStream;
-
-/**
- * JAX-RS streaming output to return the binary container data.
- */
-public class ContainerStreamingOutput implements StreamingOutput {
-
- private long containerId;
-
- private ContainerReplicationSource containerReplicationSource;
-
- public ContainerStreamingOutput(long containerId,
- ContainerReplicationSource containerReplicationSource) {
- this.containerId = containerId;
- this.containerReplicationSource = containerReplicationSource;
- }
-
- @Override
- public void write(OutputStream outputStream)
- throws IOException {
- containerReplicationSource.copyData(containerId, outputStream);
- }
-}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SimpleContainerDownloader.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SimpleContainerDownloader.java
index 37a44ac..d7666ea 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SimpleContainerDownloader.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SimpleContainerDownloader.java
@@ -24,7 +24,7 @@
import java.util.concurrent.CompletableFuture;
import java.util.function.Function;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name;
import org.apache.hadoop.ozone.OzoneConfigKeys;
@@ -46,7 +46,7 @@
private final Path workingDirectory;
- public SimpleContainerDownloader(Configuration conf) {
+ public SimpleContainerDownloader(ConfigurationSource conf) {
String workDirString =
conf.get(OzoneConfigKeys.OZONE_CONTAINER_COPY_WORKDIR);
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java
index aa28f81..e99cbae 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java
@@ -31,6 +31,7 @@
import org.apache.hadoop.hdds.utils.ProtocolMessageMetrics;
import org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol;
+import com.google.protobuf.ProtocolMessageEnum;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
import org.slf4j.Logger;
@@ -53,7 +54,7 @@
public StorageContainerDatanodeProtocolServerSideTranslatorPB(
StorageContainerDatanodeProtocol impl,
- ProtocolMessageMetrics protocolMessageMetrics) {
+ ProtocolMessageMetrics<ProtocolMessageEnum> protocolMessageMetrics) {
this.impl = impl;
dispatcher =
new OzoneProtocolMessageDispatcher<>("SCMDatanodeProtocol",
diff --git a/hadoop-hdds/container-service/src/main/proto/proto.lock b/hadoop-hdds/container-service/src/main/proto/proto.lock
new file mode 100644
index 0000000..ac272e5
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/proto/proto.lock
@@ -0,0 +1,1070 @@
+{
+ "definitions": [
+ {
+ "protopath": "StorageContainerDatanodeProtocol.proto",
+ "def": {
+ "enums": [
+ {
+ "name": "Type",
+ "enum_fields": [
+ {
+ "name": "GetVersion",
+ "integer": 1
+ },
+ {
+ "name": "Register",
+ "integer": 2
+ },
+ {
+ "name": "SendHeartbeat",
+ "integer": 3
+ }
+ ]
+ },
+ {
+ "name": "Status",
+ "enum_fields": [
+ {
+ "name": "OK",
+ "integer": 1
+ },
+ {
+ "name": "ERROR",
+ "integer": 2
+ }
+ ]
+ },
+ {
+ "name": "SCMRegisteredResponseProto.ErrorCode",
+ "enum_fields": [
+ {
+ "name": "success",
+ "integer": 1
+ },
+ {
+ "name": "errorNodeNotPermitted",
+ "integer": 2
+ }
+ ]
+ },
+ {
+ "name": "StorageTypeProto",
+ "enum_fields": [
+ {
+ "name": "DISK",
+ "integer": 1
+ },
+ {
+ "name": "SSD",
+ "integer": 2
+ },
+ {
+ "name": "ARCHIVE",
+ "integer": 3
+ },
+ {
+ "name": "RAM_DISK",
+ "integer": 4
+ },
+ {
+ "name": "PROVIDED",
+ "integer": 5
+ }
+ ]
+ },
+ {
+ "name": "ContainerReplicaProto.State",
+ "enum_fields": [
+ {
+ "name": "OPEN",
+ "integer": 1
+ },
+ {
+ "name": "CLOSING",
+ "integer": 2
+ },
+ {
+ "name": "QUASI_CLOSED",
+ "integer": 3
+ },
+ {
+ "name": "CLOSED",
+ "integer": 4
+ },
+ {
+ "name": "UNHEALTHY",
+ "integer": 5
+ },
+ {
+ "name": "INVALID",
+ "integer": 6
+ }
+ ]
+ },
+ {
+ "name": "CommandStatus.Status",
+ "enum_fields": [
+ {
+ "name": "PENDING",
+ "integer": 1
+ },
+ {
+ "name": "EXECUTED",
+ "integer": 2
+ },
+ {
+ "name": "FAILED",
+ "integer": 3
+ }
+ ]
+ },
+ {
+ "name": "ContainerAction.Action",
+ "enum_fields": [
+ {
+ "name": "CLOSE",
+ "integer": 1
+ }
+ ]
+ },
+ {
+ "name": "ContainerAction.Reason",
+ "enum_fields": [
+ {
+ "name": "CONTAINER_FULL",
+ "integer": 1
+ },
+ {
+ "name": "CONTAINER_UNHEALTHY",
+ "integer": 2
+ }
+ ]
+ },
+ {
+ "name": "ClosePipelineInfo.Reason",
+ "enum_fields": [
+ {
+ "name": "PIPELINE_FAILED",
+ "integer": 1
+ },
+ {
+ "name": "PIPELINE_LOG_FAILED",
+ "integer": 2
+ },
+ {
+ "name": "STATEMACHINE_TRANSACTION_FAILED",
+ "integer": 3
+ }
+ ]
+ },
+ {
+ "name": "PipelineAction.Action",
+ "enum_fields": [
+ {
+ "name": "CLOSE",
+ "integer": 1
+ }
+ ]
+ },
+ {
+ "name": "SCMCommandProto.Type",
+ "enum_fields": [
+ {
+ "name": "reregisterCommand",
+ "integer": 1
+ },
+ {
+ "name": "deleteBlocksCommand",
+ "integer": 2
+ },
+ {
+ "name": "closeContainerCommand",
+ "integer": 3
+ },
+ {
+ "name": "deleteContainerCommand",
+ "integer": 4
+ },
+ {
+ "name": "replicateContainerCommand",
+ "integer": 5
+ },
+ {
+ "name": "createPipelineCommand",
+ "integer": 6
+ },
+ {
+ "name": "closePipelineCommand",
+ "integer": 7
+ }
+ ]
+ }
+ ],
+ "messages": [
+ {
+ "name": "SCMDatanodeRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "cmdType",
+ "type": "Type"
+ },
+ {
+ "id": 2,
+ "name": "traceID",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "getVersionRequest",
+ "type": "SCMVersionRequestProto"
+ },
+ {
+ "id": 4,
+ "name": "registerRequest",
+ "type": "SCMRegisterRequestProto"
+ },
+ {
+ "id": 5,
+ "name": "sendHeartbeatRequest",
+ "type": "SCMHeartbeatRequestProto"
+ }
+ ]
+ },
+ {
+ "name": "SCMDatanodeResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "cmdType",
+ "type": "Type"
+ },
+ {
+ "id": 2,
+ "name": "traceID",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "success",
+ "type": "bool",
+ "options": [
+ {
+ "name": "default",
+ "value": "true"
+ }
+ ]
+ },
+ {
+ "id": 4,
+ "name": "message",
+ "type": "string"
+ },
+ {
+ "id": 5,
+ "name": "status",
+ "type": "Status"
+ },
+ {
+ "id": 6,
+ "name": "getVersionResponse",
+ "type": "SCMVersionResponseProto"
+ },
+ {
+ "id": 7,
+ "name": "registerResponse",
+ "type": "SCMRegisteredResponseProto"
+ },
+ {
+ "id": 8,
+ "name": "sendHeartbeatResponse",
+ "type": "SCMHeartbeatResponseProto"
+ }
+ ]
+ },
+ {
+ "name": "SCMVersionRequestProto"
+ },
+ {
+ "name": "SCMVersionResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "softwareVersion",
+ "type": "uint32"
+ },
+ {
+ "id": 2,
+ "name": "keys",
+ "type": "hadoop.hdds.KeyValue",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "SCMRegisterRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "datanodeDetails",
+ "type": "DatanodeDetailsProto"
+ },
+ {
+ "id": 2,
+ "name": "nodeReport",
+ "type": "NodeReportProto"
+ },
+ {
+ "id": 3,
+ "name": "containerReport",
+ "type": "ContainerReportsProto"
+ },
+ {
+ "id": 4,
+ "name": "pipelineReports",
+ "type": "PipelineReportsProto"
+ }
+ ]
+ },
+ {
+ "name": "SCMRegisteredResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "errorCode",
+ "type": "ErrorCode"
+ },
+ {
+ "id": 2,
+ "name": "datanodeUUID",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "clusterID",
+ "type": "string"
+ },
+ {
+ "id": 4,
+ "name": "addressList",
+ "type": "SCMNodeAddressList"
+ },
+ {
+ "id": 5,
+ "name": "hostname",
+ "type": "string"
+ },
+ {
+ "id": 6,
+ "name": "ipAddress",
+ "type": "string"
+ },
+ {
+ "id": 7,
+ "name": "networkName",
+ "type": "string"
+ },
+ {
+ "id": 8,
+ "name": "networkLocation",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "SCMHeartbeatRequestProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "datanodeDetails",
+ "type": "DatanodeDetailsProto"
+ },
+ {
+ "id": 2,
+ "name": "nodeReport",
+ "type": "NodeReportProto"
+ },
+ {
+ "id": 3,
+ "name": "containerReport",
+ "type": "ContainerReportsProto"
+ },
+ {
+ "id": 4,
+ "name": "incrementalContainerReport",
+ "type": "IncrementalContainerReportProto",
+ "is_repeated": true
+ },
+ {
+ "id": 5,
+ "name": "commandStatusReports",
+ "type": "CommandStatusReportsProto",
+ "is_repeated": true
+ },
+ {
+ "id": 6,
+ "name": "containerActions",
+ "type": "ContainerActionsProto"
+ },
+ {
+ "id": 7,
+ "name": "pipelineActions",
+ "type": "PipelineActionsProto"
+ },
+ {
+ "id": 8,
+ "name": "pipelineReports",
+ "type": "PipelineReportsProto"
+ }
+ ]
+ },
+ {
+ "name": "SCMHeartbeatResponseProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "datanodeUUID",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "commands",
+ "type": "SCMCommandProto",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "SCMNodeAddressList",
+ "fields": [
+ {
+ "id": 1,
+ "name": "addressList",
+ "type": "string",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "NodeReportProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "storageReport",
+ "type": "StorageReportProto",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "StorageReportProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "storageUuid",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "storageLocation",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "capacity",
+ "type": "uint64",
+ "options": [
+ {
+ "name": "default",
+ "value": "0"
+ }
+ ]
+ },
+ {
+ "id": 4,
+ "name": "scmUsed",
+ "type": "uint64",
+ "options": [
+ {
+ "name": "default",
+ "value": "0"
+ }
+ ]
+ },
+ {
+ "id": 5,
+ "name": "remaining",
+ "type": "uint64",
+ "options": [
+ {
+ "name": "default",
+ "value": "0"
+ }
+ ]
+ },
+ {
+ "id": 6,
+ "name": "storageType",
+ "type": "StorageTypeProto",
+ "options": [
+ {
+ "name": "default",
+ "value": "DISK"
+ }
+ ]
+ },
+ {
+ "id": 7,
+ "name": "failed",
+ "type": "bool",
+ "options": [
+ {
+ "name": "default",
+ "value": "false"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "ContainerReportsProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "reports",
+ "type": "ContainerReplicaProto",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "IncrementalContainerReportProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "report",
+ "type": "ContainerReplicaProto",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "ContainerReplicaProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "containerID",
+ "type": "int64"
+ },
+ {
+ "id": 2,
+ "name": "state",
+ "type": "State"
+ },
+ {
+ "id": 3,
+ "name": "size",
+ "type": "int64"
+ },
+ {
+ "id": 4,
+ "name": "used",
+ "type": "int64"
+ },
+ {
+ "id": 5,
+ "name": "keyCount",
+ "type": "int64"
+ },
+ {
+ "id": 6,
+ "name": "readCount",
+ "type": "int64"
+ },
+ {
+ "id": 7,
+ "name": "writeCount",
+ "type": "int64"
+ },
+ {
+ "id": 8,
+ "name": "readBytes",
+ "type": "int64"
+ },
+ {
+ "id": 9,
+ "name": "writeBytes",
+ "type": "int64"
+ },
+ {
+ "id": 10,
+ "name": "finalhash",
+ "type": "string"
+ },
+ {
+ "id": 11,
+ "name": "deleteTransactionId",
+ "type": "int64"
+ },
+ {
+ "id": 12,
+ "name": "blockCommitSequenceId",
+ "type": "uint64"
+ },
+ {
+ "id": 13,
+ "name": "originNodeId",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "CommandStatusReportsProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "cmdStatus",
+ "type": "CommandStatus",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "CommandStatus",
+ "fields": [
+ {
+ "id": 1,
+ "name": "cmdId",
+ "type": "int64"
+ },
+ {
+ "id": 2,
+ "name": "status",
+ "type": "Status",
+ "options": [
+ {
+ "name": "default",
+ "value": "PENDING"
+ }
+ ]
+ },
+ {
+ "id": 3,
+ "name": "type",
+ "type": "SCMCommandProto.Type"
+ },
+ {
+ "id": 4,
+ "name": "msg",
+ "type": "string"
+ },
+ {
+ "id": 5,
+ "name": "blockDeletionAck",
+ "type": "ContainerBlocksDeletionACKProto"
+ }
+ ]
+ },
+ {
+ "name": "ContainerActionsProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "containerActions",
+ "type": "ContainerAction",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "ContainerAction",
+ "fields": [
+ {
+ "id": 1,
+ "name": "containerID",
+ "type": "int64"
+ },
+ {
+ "id": 2,
+ "name": "action",
+ "type": "Action"
+ },
+ {
+ "id": 3,
+ "name": "reason",
+ "type": "Reason"
+ }
+ ]
+ },
+ {
+ "name": "PipelineReport",
+ "fields": [
+ {
+ "id": 1,
+ "name": "pipelineID",
+ "type": "PipelineID"
+ },
+ {
+ "id": 2,
+ "name": "isLeader",
+ "type": "bool"
+ },
+ {
+ "id": 3,
+ "name": "bytesWritten",
+ "type": "uint64"
+ }
+ ]
+ },
+ {
+ "name": "PipelineReportsProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "pipelineReport",
+ "type": "PipelineReport",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "PipelineActionsProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "pipelineActions",
+ "type": "PipelineAction",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "ClosePipelineInfo",
+ "fields": [
+ {
+ "id": 1,
+ "name": "pipelineID",
+ "type": "PipelineID"
+ },
+ {
+ "id": 3,
+ "name": "reason",
+ "type": "Reason"
+ },
+ {
+ "id": 4,
+ "name": "detailedReason",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "PipelineAction",
+ "fields": [
+ {
+ "id": 1,
+ "name": "action",
+ "type": "Action"
+ },
+ {
+ "id": 2,
+ "name": "closePipeline",
+ "type": "ClosePipelineInfo"
+ }
+ ]
+ },
+ {
+ "name": "SCMCommandProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "commandType",
+ "type": "Type"
+ },
+ {
+ "id": 2,
+ "name": "reregisterCommandProto",
+ "type": "ReregisterCommandProto"
+ },
+ {
+ "id": 3,
+ "name": "deleteBlocksCommandProto",
+ "type": "DeleteBlocksCommandProto"
+ },
+ {
+ "id": 4,
+ "name": "closeContainerCommandProto",
+ "type": "CloseContainerCommandProto"
+ },
+ {
+ "id": 5,
+ "name": "deleteContainerCommandProto",
+ "type": "DeleteContainerCommandProto"
+ },
+ {
+ "id": 6,
+ "name": "replicateContainerCommandProto",
+ "type": "ReplicateContainerCommandProto"
+ },
+ {
+ "id": 7,
+ "name": "createPipelineCommandProto",
+ "type": "CreatePipelineCommandProto"
+ },
+ {
+ "id": 8,
+ "name": "closePipelineCommandProto",
+ "type": "ClosePipelineCommandProto"
+ }
+ ]
+ },
+ {
+ "name": "ReregisterCommandProto"
+ },
+ {
+ "name": "DeleteBlocksCommandProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "deletedBlocksTransactions",
+ "type": "DeletedBlocksTransaction",
+ "is_repeated": true
+ },
+ {
+ "id": 3,
+ "name": "cmdId",
+ "type": "int64"
+ }
+ ]
+ },
+ {
+ "name": "DeletedBlocksTransaction",
+ "fields": [
+ {
+ "id": 1,
+ "name": "txID",
+ "type": "int64"
+ },
+ {
+ "id": 2,
+ "name": "containerID",
+ "type": "int64"
+ },
+ {
+ "id": 3,
+ "name": "localID",
+ "type": "int64",
+ "is_repeated": true
+ },
+ {
+ "id": 4,
+ "name": "count",
+ "type": "int32"
+ }
+ ]
+ },
+ {
+ "name": "ContainerBlocksDeletionACKProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "results",
+ "type": "DeleteBlockTransactionResult",
+ "is_repeated": true
+ },
+ {
+ "id": 2,
+ "name": "dnId",
+ "type": "string"
+ }
+ ],
+ "messages": [
+ {
+ "name": "DeleteBlockTransactionResult",
+ "fields": [
+ {
+ "id": 1,
+ "name": "txID",
+ "type": "int64"
+ },
+ {
+ "id": 2,
+ "name": "containerID",
+ "type": "int64"
+ },
+ {
+ "id": 3,
+ "name": "success",
+ "type": "bool"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "CloseContainerCommandProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "containerID",
+ "type": "int64"
+ },
+ {
+ "id": 2,
+ "name": "pipelineID",
+ "type": "PipelineID"
+ },
+ {
+ "id": 3,
+ "name": "cmdId",
+ "type": "int64"
+ },
+ {
+ "id": 4,
+ "name": "force",
+ "type": "bool",
+ "options": [
+ {
+ "name": "default",
+ "value": "false"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "DeleteContainerCommandProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "containerID",
+ "type": "int64"
+ },
+ {
+ "id": 2,
+ "name": "cmdId",
+ "type": "int64"
+ },
+ {
+ "id": 3,
+ "name": "force",
+ "type": "bool",
+ "options": [
+ {
+ "name": "default",
+ "value": "false"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "ReplicateContainerCommandProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "containerID",
+ "type": "int64"
+ },
+ {
+ "id": 2,
+ "name": "sources",
+ "type": "DatanodeDetailsProto",
+ "is_repeated": true
+ },
+ {
+ "id": 3,
+ "name": "cmdId",
+ "type": "int64"
+ }
+ ]
+ },
+ {
+ "name": "CreatePipelineCommandProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "pipelineID",
+ "type": "PipelineID"
+ },
+ {
+ "id": 2,
+ "name": "type",
+ "type": "ReplicationType"
+ },
+ {
+ "id": 3,
+ "name": "factor",
+ "type": "ReplicationFactor"
+ },
+ {
+ "id": 4,
+ "name": "datanode",
+ "type": "DatanodeDetailsProto",
+ "is_repeated": true
+ },
+ {
+ "id": 5,
+ "name": "cmdId",
+ "type": "int64"
+ }
+ ]
+ },
+ {
+ "name": "ClosePipelineCommandProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "pipelineID",
+ "type": "PipelineID"
+ },
+ {
+ "id": 2,
+ "name": "cmdId",
+ "type": "int64"
+ }
+ ]
+ }
+ ],
+ "services": [
+ {
+ "name": "StorageContainerDatanodeProtocolService",
+ "rpcs": [
+ {
+ "name": "submitRequest",
+ "in_type": "SCMDatanodeRequest",
+ "out_type": "SCMDatanodeResponse"
+ }
+ ]
+ }
+ ],
+ "imports": [
+ {
+ "path": "hdds.proto"
+ }
+ ],
+ "package": {
+ "name": "hadoop.hdds"
+ },
+ "options": [
+ {
+ "name": "java_package",
+ "value": "org.apache.hadoop.hdds.protocol.proto"
+ },
+ {
+ "name": "java_outer_classname",
+ "value": "StorageContainerDatanodeProtocolProtos"
+ },
+ {
+ "name": "java_generic_services",
+ "value": "true"
+ },
+ {
+ "name": "java_generate_equals_and_hash",
+ "value": "true"
+ }
+ ]
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java
index 6f159b4..aabde54 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java
@@ -17,33 +17,33 @@
package org.apache.hadoop.ozone.container.common;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.Random;
+import java.util.UUID;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.ozone.container.common.impl.ChunkLayOutVersion;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
-import org.apache.hadoop.ozone.container.common.statemachine
- .EndpointStateMachine;
+import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
-import org.apache.hadoop.ozone.protocolPB
- .StorageContainerDatanodeProtocolClientSideTranslatorPB;
+import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolClientSideTranslatorPB;
import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolPB;
import org.apache.hadoop.security.UserGroupInformation;
-import org.mockito.Mockito;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.Random;
-import java.util.UUID;
+import org.mockito.Mockito;
/**
* Helper utility to test containers.
@@ -77,7 +77,8 @@
StorageContainerDatanodeProtocolClientSideTranslatorPB rpcClient =
new StorageContainerDatanodeProtocolClientSideTranslatorPB(rpcProxy);
- return new EndpointStateMachine(address, rpcClient, conf);
+ return new EndpointStateMachine(address, rpcClient,
+ new LegacyHadoopConfigurationSource(conf));
}
public static OzoneContainer getOzoneContainer(
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/SCMTestUtils.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/SCMTestUtils.java
index b2d412a..f696ac3 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/SCMTestUtils.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/SCMTestUtils.java
@@ -22,18 +22,20 @@
import java.net.ServerSocket;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.fs.SpaceUsageCheckFactory;
-import org.apache.hadoop.hdds.fs.MockSpaceUsageCheckFactory;
import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.fs.MockSpaceUsageCheckFactory;
+import org.apache.hadoop.hdds.fs.SpaceUsageCheckFactory;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageContainerDatanodeProtocolService;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
+import org.apache.hadoop.hdds.utils.ProtocolMessageMetrics;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol;
-import org.apache.hadoop.hdds.utils.ProtocolMessageMetrics;
import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolPB;
import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolServerSideTranslatorPB;
import org.apache.hadoop.test.GenericTestUtils;
@@ -85,11 +87,14 @@
/**
* Start Datanode RPC server.
*/
- public static RPC.Server startScmRpcServer(Configuration configuration,
+ public static RPC.Server startScmRpcServer(ConfigurationSource configuration,
StorageContainerDatanodeProtocol server,
InetSocketAddress rpcServerAddresss, int handlerCount) throws
IOException {
- RPC.setProtocolEngine(configuration,
+
+ Configuration hadoopConfig =
+ LegacyHadoopConfigurationSource.asHadoopConfiguration(configuration);
+ RPC.setProtocolEngine(hadoopConfig,
StorageContainerDatanodeProtocolPB.class,
ProtobufRpcEngine.class);
@@ -99,7 +104,7 @@
new StorageContainerDatanodeProtocolServerSideTranslatorPB(
server, Mockito.mock(ProtocolMessageMetrics.class)));
- RPC.Server scmServer = startRpcServer(configuration, rpcServerAddresss,
+ RPC.Server scmServer = startRpcServer(hadoopConfig, rpcServerAddresss,
StorageContainerDatanodeProtocolPB.class, scmDatanodeService,
handlerCount);
@@ -133,20 +138,20 @@
}
public static HddsProtos.ReplicationType getReplicationType(
- Configuration conf) {
+ ConfigurationSource conf) {
return isUseRatis(conf) ?
HddsProtos.ReplicationType.RATIS :
HddsProtos.ReplicationType.STAND_ALONE;
}
public static HddsProtos.ReplicationFactor getReplicationFactor(
- Configuration conf) {
+ ConfigurationSource conf) {
return isUseRatis(conf) ?
HddsProtos.ReplicationFactor.THREE :
HddsProtos.ReplicationFactor.ONE;
}
- private static boolean isUseRatis(Configuration c) {
+ private static boolean isUseRatis(ConfigurationSource c) {
return c.getBoolean(
ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY,
ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT);
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
index 303ebd7..b4c1ae5 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
@@ -17,13 +17,27 @@
package org.apache.hadoop.ozone.container.common;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
import com.google.common.collect.Lists;
+import com.google.common.primitives.Ints;
+import com.google.common.primitives.Longs;
import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.client.BlockID;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.MetadataKeyFilters;
+import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.common.Checksum;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
@@ -35,40 +49,34 @@
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
import org.apache.hadoop.ozone.container.common.interfaces.Handler;
-import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
+import org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB;
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
+import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.keyvalue.ChunkLayoutTestInfo;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
+import org.apache.hadoop.ozone.container.keyvalue.statemachine.background.BlockDeletingService;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.container.testutils.BlockDeletingServiceTestImpl;
-import org.apache.hadoop.ozone.container.keyvalue.statemachine.background.BlockDeletingService;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
-import org.apache.hadoop.hdds.utils.BackgroundService;
-import org.apache.hadoop.hdds.utils.MetadataKeyFilters;
-import org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB;
+
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
import org.junit.AfterClass;
import org.junit.Assert;
-import org.junit.Test;
import org.junit.BeforeClass;
+import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL;
+import static org.apache.hadoop.ozone.OzoneConsts.DB_BLOCK_COUNT_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.DB_PENDING_DELETE_BLOCK_COUNT_KEY;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
@@ -119,7 +127,8 @@
* creates some fake chunk files for testing.
*/
private void createToDeleteBlocks(ContainerSet containerSet,
- Configuration conf, int numOfContainers, int numOfBlocksPerContainer,
+ ConfigurationSource conf, int numOfContainers,
+ int numOfBlocksPerContainer,
int numOfChunksPerBlock) throws IOException {
for (int x = 0; x < numOfContainers; x++) {
conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, testRoot.getAbsolutePath());
@@ -135,6 +144,8 @@
containerSet.addContainer(container);
data = (KeyValueContainerData) containerSet.getContainer(
containerID).getContainerData();
+
+ long blockLength = 100;
try(ReferenceCountedDB metadata = BlockUtils.getDB(data, conf)) {
for (int j = 0; j < numOfBlocksPerContainer; j++) {
BlockID blockID =
@@ -147,7 +158,7 @@
ContainerProtos.ChunkInfo info =
ContainerProtos.ChunkInfo.newBuilder()
.setChunkName(blockID.getLocalID() + "_chunk_" + k)
- .setLen(0)
+ .setLen(blockLength)
.setOffset(0)
.setChecksumData(Checksum.getNoChecksumDataProto())
.build();
@@ -158,6 +169,17 @@
kd.getProtoBufMessage().toByteArray());
container.getContainerData().incrPendingDeletionBlocks(1);
}
+
+ container.getContainerData().setKeyCount(numOfBlocksPerContainer);
+ container.getContainerData().setBytesUsed(
+ blockLength * numOfBlocksPerContainer);
+ // Set block count, bytes used and pending delete block count.
+ metadata.getStore().put(DB_BLOCK_COUNT_KEY,
+ Longs.toByteArray(numOfBlocksPerContainer));
+ metadata.getStore().put(OzoneConsts.DB_CONTAINER_BYTES_USED_KEY,
+ Longs.toByteArray(blockLength * numOfBlocksPerContainer));
+ metadata.getStore().put(DB_PENDING_DELETE_BLOCK_COUNT_KEY,
+ Ints.toByteArray(numOfBlocksPerContainer));
}
}
}
@@ -181,7 +203,7 @@
List<Map.Entry<byte[], byte[]>> underDeletionBlocks =
meta.getStore().getRangeKVs(null, 100,
new MetadataKeyFilters.KeyPrefixFilter()
- .addFilter(OzoneConsts.DELETING_KEY_PREFIX));
+ .addFilter(OzoneConsts.DELETING_KEY_PREFIX));
return underDeletionBlocks.size();
}
@@ -195,7 +217,7 @@
@Test
public void testBlockDeletion() throws Exception {
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
conf.setInt(OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL, 10);
conf.setInt(OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER, 2);
ContainerSet containerSet = new ContainerSet();
@@ -242,6 +264,14 @@
deleteAndWait(svc, 3);
Assert.assertEquals(0, getUnderDeletionBlocksCount(meta));
Assert.assertEquals(3, getDeletedBlocksCount(meta));
+
+
+ // Check finally DB counters.
+ // Not checking bytes used, as handler is a mock call.
+ Assert.assertEquals(0, Ints.fromByteArray(
+ meta.getStore().get(DB_PENDING_DELETE_BLOCK_COUNT_KEY)));
+ Assert.assertEquals(0, Longs.fromByteArray(
+ meta.getStore().get(DB_BLOCK_COUNT_KEY)));
}
svc.shutdown();
@@ -250,7 +280,7 @@
@Test
@SuppressWarnings("java:S2699") // waitFor => assertion with timeout
public void testShutdownService() throws Exception {
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 500,
TimeUnit.MILLISECONDS);
conf.setInt(OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL, 10);
@@ -275,7 +305,7 @@
@Test
public void testBlockDeletionTimeout() throws Exception {
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
conf.setInt(OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL, 10);
conf.setInt(OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER, 2);
ContainerSet containerSet = new ContainerSet();
@@ -336,7 +366,7 @@
}
private BlockDeletingServiceTestImpl getBlockDeletingService(
- ContainerSet containerSet, Configuration conf) {
+ ContainerSet containerSet, ConfigurationSource conf) {
OzoneContainer ozoneContainer = mockDependencies(containerSet);
return new BlockDeletingServiceTestImpl(ozoneContainer, 1000, conf);
}
@@ -363,7 +393,7 @@
//
// Each time only 1 container can be processed, so each time
// 1 block from 1 container can be deleted.
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
// Process 1 container per interval
conf.set(
ScmConfigKeys.OZONE_SCM_KEY_VALUE_CONTAINER_DELETION_CHOOSING_POLICY,
@@ -406,7 +436,7 @@
// Each time containers can be all scanned, but only 2 blocks
// per container can be actually deleted. So it requires 2 waves
// to cleanup all blocks.
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
conf.setInt(OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL, 10);
int blockLimitPerTask = 2;
conf.setInt(OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER, blockLimitPerTask);
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
index 0f3e7d1..65ae6ce 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
@@ -16,37 +16,6 @@
*/
package org.apache.hadoop.ozone.container.common;
-import com.google.common.collect.Maps;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.hadoop.ozone.container.common.statemachine
- .DatanodeStateMachine;
-import org.apache.hadoop.ozone.container.common.statemachine
- .EndpointStateMachine;
-import org.apache.hadoop.ozone.container.common.statemachine
- .SCMConnectionManager;
-import org.apache.hadoop.ozone.container.common.states.DatanodeState;
-import org.apache.hadoop.ozone.container.common.states.datanode
- .InitDatanodeState;
-import org.apache.hadoop.ozone.container.common.states.datanode
- .RunningDatanodeState;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.util.concurrent.HadoopExecutors;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -59,10 +28,35 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
+import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine;
+import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
+import org.apache.hadoop.ozone.container.common.states.DatanodeState;
+import org.apache.hadoop.ozone.container.common.states.datanode.InitDatanodeState;
+import org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
+
+import com.google.common.collect.Maps;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_HEARTBEAT_RPC_TIMEOUT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_RPC_TIMEOUT;
+import org.junit.After;
+import org.junit.Assert;
import static org.junit.Assert.assertTrue;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Tests the datanode state machine class and its states.
@@ -77,7 +71,7 @@
private List<RPC.Server> scmServers;
private List<ScmTestMock> mockServers;
private ExecutorService executorService;
- private Configuration conf;
+ private OzoneConfiguration conf;
private File testRoot;
@Before
@@ -403,7 +397,7 @@
ScmConfigKeys.OZONE_SCM_DATANODE_ID_DIR, ""));
confList.forEach((entry) -> {
- Configuration perTestConf = new Configuration(conf);
+ OzoneConfiguration perTestConf = new OzoneConfiguration(conf);
perTestConf.setStrings(entry.getKey(), entry.getValue());
LOG.info("Test with {} = {}", entry.getKey(), entry.getValue());
try (DatanodeStateMachine stateMachine = new DatanodeStateMachine(
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java
index 324ab71..7b41d99 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java
@@ -18,9 +18,9 @@
package org.apache.hadoop.ozone.container.common.interfaces;
-import com.google.common.collect.Maps;
+import java.util.Map;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
@@ -29,10 +29,11 @@
import org.apache.hadoop.ozone.container.common.impl.TestHddsDispatcher;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
+import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
+import com.google.common.collect.Maps;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -42,8 +43,6 @@
import org.junit.rules.Timeout;
import org.mockito.Mockito;
-import java.util.Map;
-
/**
* Tests Handler interface.
*/
@@ -51,7 +50,7 @@
@Rule
public TestRule timeout = new Timeout(300000);
- private Configuration conf;
+ private OzoneConfiguration conf;
private HddsDispatcher dispatcher;
private ContainerSet containerSet;
private VolumeSet volumeSet;
@@ -59,7 +58,7 @@
@Before
public void setup() throws Exception {
- this.conf = new Configuration();
+ this.conf = new OzoneConfiguration();
this.containerSet = Mockito.mock(ContainerSet.class);
this.volumeSet = Mockito.mock(MutableVolumeSet.class);
DatanodeDetails datanodeDetails = Mockito.mock(DatanodeDetails.class);
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportManager.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportManager.java
index aae388d..45e50dc 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportManager.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportManager.java
@@ -17,16 +17,15 @@
package org.apache.hadoop.ozone.container.common.report;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.junit.Test;
-import org.mockito.Mockito;
-
import java.util.concurrent.ScheduledExecutorService;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+
+import org.junit.Test;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
+import org.mockito.Mockito;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -37,7 +36,7 @@
@Test
public void testReportManagerInit() {
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
StateContext dummyContext = Mockito.mock(StateContext.class);
ReportPublisher dummyPublisher = Mockito.mock(ReportPublisher.class);
ReportManager.Builder builder = ReportManager.newBuilder(conf);
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java
index 03f0cd4..166aadf 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java
@@ -21,7 +21,7 @@
import com.google.protobuf.GeneratedMessage;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.HddsIdFactory;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -51,7 +51,7 @@
*/
public class TestReportPublisher {
- private static Configuration config;
+ private static ConfigurationSource config;
@BeforeClass
public static void setup() {
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisherFactory.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisherFactory.java
index f8c5fe5..e9a34c7 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisherFactory.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisherFactory.java
@@ -17,13 +17,11 @@
package org.apache.hadoop.ozone.container.common.report;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.NodeReportProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
+
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
@@ -39,7 +37,7 @@
@Test
public void testGetContainerReportPublisher() {
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
ReportPublisherFactory factory = new ReportPublisherFactory(conf);
ReportPublisher publisher = factory
.getPublisherFor(ContainerReportsProto.class);
@@ -49,7 +47,7 @@
@Test
public void testGetNodeReportPublisher() {
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
ReportPublisherFactory factory = new ReportPublisherFactory(conf);
ReportPublisher publisher = factory
.getPublisherFor(NodeReportProto.class);
@@ -59,7 +57,7 @@
@Test
public void testInvalidReportPublisher() {
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
ReportPublisherFactory factory = new ReportPublisherFactory(conf);
exception.expect(RuntimeException.class);
exception.expectMessage("No publisher found for report");
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/TestHeartbeatEndpointTask.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/TestHeartbeatEndpointTask.java
index 95ac87f..1c66d63 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/TestHeartbeatEndpointTask.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/TestHeartbeatEndpointTask.java
@@ -18,39 +18,29 @@
package org.apache.hadoop.ozone.container.common.states.endpoint;
-import org.apache.hadoop.conf.Configuration;
+import java.net.InetSocketAddress;
+import java.util.UUID;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerAction;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.CommandStatusReportsProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.NodeReportProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMHeartbeatResponseProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
-import org.apache.hadoop.ozone.container.common.statemachine
- .DatanodeStateMachine;
-import org.apache.hadoop.ozone.container.common.statemachine
- .DatanodeStateMachine.DatanodeStates;
-import org.apache.hadoop.ozone.container.common.statemachine
- .EndpointStateMachine;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandStatusReportsProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerAction;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMHeartbeatResponseProto;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.DatanodeStates;
+import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.protocolPB
- .StorageContainerDatanodeProtocolClientSideTranslatorPB;
+import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolClientSideTranslatorPB;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
-import java.net.InetSocketAddress;
-import java.util.UUID;
-
/**
* This class tests the functionality of HeartbeatEndpointTask.
*/
@@ -86,7 +76,7 @@
@Test
public void testheartbeatWithNodeReports() throws Exception {
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
StateContext context = new StateContext(conf, DatanodeStates.RUNNING,
Mockito.mock(DatanodeStateMachine.class));
@@ -118,7 +108,7 @@
@Test
public void testheartbeatWithContainerReports() throws Exception {
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
StateContext context = new StateContext(conf, DatanodeStates.RUNNING,
Mockito.mock(DatanodeStateMachine.class));
@@ -150,7 +140,7 @@
@Test
public void testheartbeatWithCommandStatusReports() throws Exception {
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
StateContext context = new StateContext(conf, DatanodeStates.RUNNING,
Mockito.mock(DatanodeStateMachine.class));
@@ -182,7 +172,7 @@
@Test
public void testheartbeatWithContainerActions() throws Exception {
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
StateContext context = new StateContext(conf, DatanodeStates.RUNNING,
Mockito.mock(DatanodeStateMachine.class));
@@ -214,7 +204,7 @@
@Test
public void testheartbeatWithAllReports() throws Exception {
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
StateContext context = new StateContext(conf, DatanodeStates.RUNNING,
Mockito.mock(DatanodeStateMachine.class));
@@ -256,7 +246,7 @@
*/
private HeartbeatEndpointTask getHeartbeatEndpointTask(
StorageContainerDatanodeProtocolClientSideTranslatorPB proxy) {
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
StateContext context = new StateContext(conf, DatanodeStates.RUNNING,
Mockito.mock(DatanodeStateMachine.class));
return getHeartbeatEndpointTask(conf, context, proxy);
@@ -274,7 +264,7 @@
* @return HeartbeatEndpointTask
*/
private HeartbeatEndpointTask getHeartbeatEndpointTask(
- Configuration conf,
+ ConfigurationSource conf,
StateContext context,
StorageContainerDatanodeProtocolClientSideTranslatorPB proxy) {
DatanodeDetails datanodeDetails = DatanodeDetails.newBuilder()
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java
index 0d9c876..57a0e55 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java
@@ -17,31 +17,31 @@
package org.apache.hadoop.ozone.container.common.volume;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.fs.SpaceUsageCheckFactory;
-import org.apache.hadoop.hdds.fs.SpaceUsagePersistence;
-import org.apache.hadoop.hdds.fs.SpaceUsageSource;
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.hdds.fs.MockSpaceUsageCheckFactory;
-import org.apache.hadoop.ozone.container.common.helpers.DatanodeVersionFile;
-import org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
import java.io.File;
import java.time.Duration;
import java.util.Properties;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.fs.MockSpaceUsageCheckFactory;
+import org.apache.hadoop.hdds.fs.SpaceUsageCheckFactory;
+import org.apache.hadoop.hdds.fs.SpaceUsagePersistence;
+import org.apache.hadoop.hdds.fs.SpaceUsageSource;
+import org.apache.hadoop.ozone.container.common.helpers.DatanodeVersionFile;
+import org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil;
+
import static org.apache.hadoop.hdds.fs.MockSpaceUsagePersistence.inMemory;
import static org.apache.hadoop.hdds.fs.MockSpaceUsageSource.fixed;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
/**
* Unit tests for {@link HddsVolume}.
@@ -50,7 +50,7 @@
private static final String DATANODE_UUID = UUID.randomUUID().toString();
private static final String CLUSTER_ID = UUID.randomUUID().toString();
- private static final Configuration CONF = new Configuration();
+ private static final OzoneConfiguration CONF = new OzoneConfiguration();
@Rule
public TemporaryFolder folder = new TemporaryFolder();
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestRoundRobinVolumeChoosingPolicy.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestRoundRobinVolumeChoosingPolicy.java
index 19ee54d..b2a39a9 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestRoundRobinVolumeChoosingPolicy.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestRoundRobinVolumeChoosingPolicy.java
@@ -18,24 +18,24 @@
package org.apache.hadoop.ozone.container.common.volume;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.fs.SpaceUsageCheckFactory;
-import org.apache.hadoop.hdds.fs.SpaceUsagePersistence;
-import org.apache.hadoop.hdds.fs.SpaceUsageSource;
-import org.apache.hadoop.hdds.fs.MockSpaceUsageCheckFactory;
-import org.apache.hadoop.hdds.fs.MockSpaceUsageSource;
-import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
import java.io.IOException;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.fs.MockSpaceUsageCheckFactory;
+import org.apache.hadoop.hdds.fs.MockSpaceUsageSource;
+import org.apache.hadoop.hdds.fs.SpaceUsageCheckFactory;
+import org.apache.hadoop.hdds.fs.SpaceUsagePersistence;
+import org.apache.hadoop.hdds.fs.SpaceUsageSource;
+import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
+
import static org.apache.hadoop.test.GenericTestUtils.getTestDir;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
/**
* Tests {@link RoundRobinVolumeChoosingPolicy}.
@@ -45,7 +45,7 @@
private RoundRobinVolumeChoosingPolicy policy;
private final List<HddsVolume> volumes = new ArrayList<>();
- private static final Configuration CONF = new Configuration();
+ private static final OzoneConfiguration CONF = new OzoneConfiguration();
private static final String BASE_DIR =
getTestDir(TestRoundRobinVolumeChoosingPolicy.class.getSimpleName())
.getAbsolutePath();
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSetDiskChecks.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSetDiskChecks.java
index c8a8114..3bd42d8 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSetDiskChecks.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSetDiskChecks.java
@@ -26,16 +26,16 @@
import java.util.Set;
import java.util.UUID;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.DFSConfigKeysLegacy;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.DiskChecker.DiskErrorException;
import org.apache.hadoop.util.Timer;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import org.apache.commons.io.FileUtils;
-import org.apache.curator.shaded.com.google.common.collect.ImmutableSet;
import static org.hamcrest.CoreMatchers.is;
import org.junit.After;
import static org.junit.Assert.assertEquals;
@@ -63,7 +63,7 @@
@Rule
public ExpectedException thrown = ExpectedException.none();
- private Configuration conf = null;
+ private OzoneConfiguration conf = null;
/**
* Cleanup volume directories.
@@ -115,7 +115,7 @@
final MutableVolumeSet volumeSet = new MutableVolumeSet(
UUID.randomUUID().toString(), conf) {
@Override
- HddsVolumeChecker getVolumeChecker(Configuration configuration)
+ HddsVolumeChecker getVolumeChecker(ConfigurationSource configuration)
throws DiskErrorException {
return new DummyChecker(configuration, new Timer(), numBadVolumes);
}
@@ -139,7 +139,7 @@
final MutableVolumeSet volumeSet = new MutableVolumeSet(
UUID.randomUUID().toString(), conf) {
@Override
- HddsVolumeChecker getVolumeChecker(Configuration configuration)
+ HddsVolumeChecker getVolumeChecker(ConfigurationSource configuration)
throws DiskErrorException {
return new DummyChecker(configuration, new Timer(), numVolumes);
}
@@ -155,8 +155,8 @@
* storage directories.
* @param numDirs
*/
- private Configuration getConfWithDataNodeDirs(int numDirs) {
- final Configuration ozoneConf = new OzoneConfiguration();
+ private OzoneConfiguration getConfWithDataNodeDirs(int numDirs) {
+ final OzoneConfiguration ozoneConf = new OzoneConfiguration();
final List<String> dirs = new ArrayList<>();
for (int i = 0; i < numDirs; ++i) {
dirs.add(GenericTestUtils.getRandomizedTestDir().getPath());
@@ -173,7 +173,7 @@
static class DummyChecker extends HddsVolumeChecker {
private final int numBadVolumes;
- DummyChecker(Configuration conf, Timer timer, int numBadVolumes)
+ DummyChecker(ConfigurationSource conf, Timer timer, int numBadVolumes)
throws DiskErrorException {
super(conf, timer);
this.numBadVolumes = numBadVolumes;
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java
index ab4d5f6..256f8b7 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java
@@ -18,30 +18,6 @@
package org.apache.hadoop.ozone.container.keyvalue;
-import com.google.common.primitives.Longs;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.StorageUnit;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hdds.client.BlockID;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.container.common.helpers.BlockData;
-import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
-import org.apache.hadoop.ozone.container.common.impl.ChunkLayOutVersion;
-import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
-import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
-import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.hdds.utils.MetadataKeyFilters;
-import org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
@@ -50,17 +26,38 @@
import java.util.NoSuchElementException;
import java.util.UUID;
+import org.apache.hadoop.conf.StorageUnit;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.utils.MetadataKeyFilters;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
+import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
+import org.apache.hadoop.ozone.container.common.impl.ChunkLayOutVersion;
+import org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB;
+import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
+import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
+import org.apache.hadoop.test.GenericTestUtils;
+
+import com.google.common.primitives.Longs;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_STORE_IMPL;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_METADATA_STORE_IMPL_LEVELDB;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_METADATA_STORE_IMPL_ROCKSDB;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB;
import static org.apache.hadoop.ozone.container.common.impl.ChunkLayOutVersion.FILE_PER_BLOCK;
import static org.apache.hadoop.ozone.container.common.impl.ChunkLayOutVersion.FILE_PER_CHUNK;
+import org.junit.After;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
/**
* This class is used to test KeyValue container block iterator.
@@ -71,7 +68,7 @@
private KeyValueContainer container;
private KeyValueContainerData containerData;
private MutableVolumeSet volumeSet;
- private Configuration conf;
+ private OzoneConfiguration conf;
private File testRoot;
private final String storeImpl;
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
index affa83a..8c6ff61 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
@@ -215,6 +215,11 @@
metadataStore.getStore().put(("test" + i).getBytes(UTF_8),
"test".getBytes(UTF_8));
}
+
+ // As now when we put blocks, we increment block count and update in DB.
+ // As for test, we are doing manually so adding key count to DB.
+ metadataStore.getStore().put(OzoneConsts.DB_BLOCK_COUNT_KEY,
+ Longs.toByteArray(numberOfKeysToWrite));
}
BlockUtils.removeDB(keyValueContainerData, conf);
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
index d9e7f09..9ad6e5b 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
@@ -18,51 +18,48 @@
package org.apache.hadoop.ozone.container.keyvalue;
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.UUID;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerCommandRequestProto;
-import org.apache.hadoop.hdds.scm.container.common.helpers
- .StorageContainerException;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerType;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.hdds.security.token.TokenVerifier;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ChunkLayOutVersion;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
+import org.apache.hadoop.ozone.container.common.interfaces.Handler;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext;
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
+import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.test.GenericTestUtils;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_VOLUME_CHOOSING_POLICY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY;
+import static org.junit.Assert.assertEquals;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestRule;
import org.junit.rules.Timeout;
-
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import org.mockito.Mockito;
-
-import static org.apache.hadoop.hdds.HddsConfigKeys
- .HDDS_DATANODE_VOLUME_CHOOSING_POLICY;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY;
-import static org.junit.Assert.assertEquals;
import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyLong;
+import org.mockito.Mockito;
import static org.mockito.Mockito.doCallRealMethod;
import static org.mockito.Mockito.times;
-import java.io.File;
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.UUID;
-
/**
* Unit tests for {@link KeyValueHandler}.
*/
@@ -94,28 +91,27 @@
public void setup() throws StorageContainerException {
// Create mock HddsDispatcher and KeyValueHandler.
handler = Mockito.mock(KeyValueHandler.class);
- dispatcher = Mockito.mock(HddsDispatcher.class);
- Mockito.when(dispatcher.getHandler(any())).thenReturn(handler);
- Mockito.when(dispatcher.dispatch(any(), any())).thenCallRealMethod();
- Mockito.when(dispatcher.getContainer(anyLong())).thenReturn(
- Mockito.mock(KeyValueContainer.class));
- Mockito.when(dispatcher.getMissingContainerSet())
- .thenReturn(new HashSet<>());
- Mockito.when(handler.handle(any(), any(), any())).thenCallRealMethod();
- doCallRealMethod().when(dispatcher).setMetricsForTesting(any());
- dispatcher.setMetricsForTesting(Mockito.mock(ContainerMetrics.class));
- Mockito.when(dispatcher.buildAuditMessageForFailure(any(), any(), any()))
- .thenCallRealMethod();
- Mockito.when(dispatcher.buildAuditMessageForSuccess(any(), any()))
- .thenCallRealMethod();
+
+ HashMap<ContainerType, Handler> handlers = new HashMap<>();
+ handlers.put(ContainerType.KeyValueContainer, handler);
+
+ dispatcher = new HddsDispatcher(
+ new OzoneConfiguration(),
+ Mockito.mock(ContainerSet.class),
+ Mockito.mock(VolumeSet.class),
+ handlers,
+ Mockito.mock(StateContext.class),
+ Mockito.mock(ContainerMetrics.class),
+ Mockito.mock(TokenVerifier.class)
+ );
+
}
/**
* Test that Handler handles different command types correctly.
*/
- @Test
- public void testHandlerCommandHandling() {
-
+ public void testHandlerCommandHandling() throws Exception {
+ Mockito.reset(handler);
// Test Create Container Request handling
ContainerCommandRequestProto createContainerRequest =
ContainerProtos.ContainerCommandRequestProto.newBuilder()
@@ -125,113 +121,132 @@
.setCreateContainer(ContainerProtos.CreateContainerRequestProto
.getDefaultInstance())
.build();
+
+ KeyValueContainer container = Mockito.mock(KeyValueContainer.class);
+
DispatcherContext context = new DispatcherContext.Builder().build();
- dispatcher.dispatch(createContainerRequest, context);
+ KeyValueHandler
+ .dispatchRequest(handler, createContainerRequest, container, context);
Mockito.verify(handler, times(1)).handleCreateContainer(
any(ContainerCommandRequestProto.class), any());
// Test Read Container Request handling
ContainerCommandRequestProto readContainerRequest =
getDummyCommandRequestProto(ContainerProtos.Type.ReadContainer);
- dispatcher.dispatch(readContainerRequest, context);
+ KeyValueHandler
+ .dispatchRequest(handler, readContainerRequest, container, context);
Mockito.verify(handler, times(1)).handleReadContainer(
any(ContainerCommandRequestProto.class), any());
// Test Update Container Request handling
ContainerCommandRequestProto updateContainerRequest =
getDummyCommandRequestProto(ContainerProtos.Type.UpdateContainer);
- dispatcher.dispatch(updateContainerRequest, context);
+ KeyValueHandler
+ .dispatchRequest(handler, updateContainerRequest, container, context);
Mockito.verify(handler, times(1)).handleUpdateContainer(
any(ContainerCommandRequestProto.class), any());
// Test Delete Container Request handling
ContainerCommandRequestProto deleteContainerRequest =
getDummyCommandRequestProto(ContainerProtos.Type.DeleteContainer);
- dispatcher.dispatch(deleteContainerRequest, null);
+ KeyValueHandler
+ .dispatchRequest(handler, deleteContainerRequest, container, context);
Mockito.verify(handler, times(1)).handleDeleteContainer(
any(ContainerCommandRequestProto.class), any());
// Test List Container Request handling
ContainerCommandRequestProto listContainerRequest =
getDummyCommandRequestProto(ContainerProtos.Type.ListContainer);
- dispatcher.dispatch(listContainerRequest, context);
+ KeyValueHandler
+ .dispatchRequest(handler, listContainerRequest, container, context);
Mockito.verify(handler, times(1)).handleUnsupportedOp(
any(ContainerCommandRequestProto.class));
// Test Close Container Request handling
ContainerCommandRequestProto closeContainerRequest =
getDummyCommandRequestProto(ContainerProtos.Type.CloseContainer);
- dispatcher.dispatch(closeContainerRequest, context);
+ KeyValueHandler
+ .dispatchRequest(handler, closeContainerRequest, container, context);
Mockito.verify(handler, times(1)).handleCloseContainer(
any(ContainerCommandRequestProto.class), any());
// Test Put Block Request handling
ContainerCommandRequestProto putBlockRequest =
getDummyCommandRequestProto(ContainerProtos.Type.PutBlock);
- dispatcher.dispatch(putBlockRequest, context);
+ KeyValueHandler
+ .dispatchRequest(handler, putBlockRequest, container, context);
Mockito.verify(handler, times(1)).handlePutBlock(
any(ContainerCommandRequestProto.class), any(), any());
// Test Get Block Request handling
ContainerCommandRequestProto getBlockRequest =
getDummyCommandRequestProto(ContainerProtos.Type.GetBlock);
- dispatcher.dispatch(getBlockRequest, context);
+ KeyValueHandler
+ .dispatchRequest(handler, getBlockRequest, container, context);
Mockito.verify(handler, times(1)).handleGetBlock(
any(ContainerCommandRequestProto.class), any());
// Test Delete Block Request handling
ContainerCommandRequestProto deleteBlockRequest =
getDummyCommandRequestProto(ContainerProtos.Type.DeleteBlock);
- dispatcher.dispatch(deleteBlockRequest, context);
+ KeyValueHandler
+ .dispatchRequest(handler, deleteBlockRequest, container, context);
Mockito.verify(handler, times(1)).handleDeleteBlock(
any(ContainerCommandRequestProto.class), any());
// Test List Block Request handling
ContainerCommandRequestProto listBlockRequest =
getDummyCommandRequestProto(ContainerProtos.Type.ListBlock);
- dispatcher.dispatch(listBlockRequest, context);
+ KeyValueHandler
+ .dispatchRequest(handler, listBlockRequest, container, context);
Mockito.verify(handler, times(2)).handleUnsupportedOp(
any(ContainerCommandRequestProto.class));
// Test Read Chunk Request handling
ContainerCommandRequestProto readChunkRequest =
getDummyCommandRequestProto(ContainerProtos.Type.ReadChunk);
- dispatcher.dispatch(readChunkRequest, context);
+ KeyValueHandler
+ .dispatchRequest(handler, readChunkRequest, container, context);
Mockito.verify(handler, times(1)).handleReadChunk(
any(ContainerCommandRequestProto.class), any(), any());
// Test Delete Chunk Request handling
ContainerCommandRequestProto deleteChunkRequest =
getDummyCommandRequestProto(ContainerProtos.Type.DeleteChunk);
- dispatcher.dispatch(deleteChunkRequest, context);
+ KeyValueHandler
+ .dispatchRequest(handler, deleteChunkRequest, container, context);
Mockito.verify(handler, times(1)).handleDeleteChunk(
any(ContainerCommandRequestProto.class), any());
// Test Write Chunk Request handling
ContainerCommandRequestProto writeChunkRequest =
getDummyCommandRequestProto(ContainerProtos.Type.WriteChunk);
- dispatcher.dispatch(writeChunkRequest, context);
+ KeyValueHandler
+ .dispatchRequest(handler, writeChunkRequest, container, context);
Mockito.verify(handler, times(1)).handleWriteChunk(
any(ContainerCommandRequestProto.class), any(), any());
// Test List Chunk Request handling
ContainerCommandRequestProto listChunkRequest =
getDummyCommandRequestProto(ContainerProtos.Type.ListChunk);
- dispatcher.dispatch(listChunkRequest, context);
+ KeyValueHandler
+ .dispatchRequest(handler, listChunkRequest, container, context);
Mockito.verify(handler, times(3)).handleUnsupportedOp(
any(ContainerCommandRequestProto.class));
// Test Put Small File Request handling
ContainerCommandRequestProto putSmallFileRequest =
getDummyCommandRequestProto(ContainerProtos.Type.PutSmallFile);
- dispatcher.dispatch(putSmallFileRequest, context);
+ KeyValueHandler
+ .dispatchRequest(handler, putSmallFileRequest, container, context);
Mockito.verify(handler, times(1)).handlePutSmallFile(
any(ContainerCommandRequestProto.class), any(), any());
// Test Get Small File Request handling
ContainerCommandRequestProto getSmallFileRequest =
getDummyCommandRequestProto(ContainerProtos.Type.GetSmallFile);
- dispatcher.dispatch(getSmallFileRequest, context);
+ KeyValueHandler
+ .dispatchRequest(handler, getSmallFileRequest, container, context);
Mockito.verify(handler, times(1)).handleGetSmallFile(
any(ContainerCommandRequestProto.class), any());
}
@@ -239,7 +254,7 @@
@Test
public void testVolumeSetInKeyValueHandler() throws Exception{
File path = GenericTestUtils.getRandomizedTestDir();
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
conf.set(HDDS_DATANODE_DIR_KEY, path.getAbsolutePath());
MutableVolumeSet
volumeSet = new MutableVolumeSet(UUID.randomUUID().toString(), conf);
@@ -296,7 +311,7 @@
@Test
public void testCloseInvalidContainer() throws IOException {
long containerID = 1234L;
- Configuration conf = new Configuration();
+ OzoneConfiguration conf = new OzoneConfiguration();
KeyValueContainerData kvData = new KeyValueContainerData(containerID,
layout,
(long) StorageUnit.GB.toBytes(1), UUID.randomUUID().toString(),
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java
new file mode 100644
index 0000000..6929864
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.ozoneimpl;
+
+import com.google.common.primitives.Longs;
+import org.apache.hadoop.conf.StorageUnit;
+import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
+import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
+import org.apache.hadoop.ozone.container.common.impl.ChunkLayOutVersion;
+import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB;
+import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
+import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
+import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.mockito.Mockito;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+
+import static org.apache.hadoop.ozone.OzoneConsts.DB_BLOCK_COUNT_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.DB_CONTAINER_BYTES_USED_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.DB_PENDING_DELETE_BLOCK_COUNT_KEY;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.mock;
+
+/**
+ * Test ContainerReader class which loads containers from disks.
+ */
+public class TestContainerReader {
+
+ @Rule
+ public final TemporaryFolder tempDir = new TemporaryFolder();
+
+ private MutableVolumeSet volumeSet;
+ private HddsVolume hddsVolume;
+ private ContainerSet containerSet;
+ private ConfigurationSource conf;
+
+
+ private RoundRobinVolumeChoosingPolicy volumeChoosingPolicy;
+ private UUID datanodeId;
+ private String scmId = UUID.randomUUID().toString();
+ private int blockCount = 10;
+ private long blockLen = 1024;
+
+ @Before
+ public void setup() throws Exception {
+
+ File volumeDir = tempDir.newFolder();
+ volumeSet = Mockito.mock(MutableVolumeSet.class);
+ containerSet = new ContainerSet();
+ conf = new OzoneConfiguration();
+
+ datanodeId = UUID.randomUUID();
+ hddsVolume = new HddsVolume.Builder(volumeDir
+ .getAbsolutePath()).conf(conf).datanodeUuid(datanodeId
+ .toString()).build();
+
+ volumeSet = mock(MutableVolumeSet.class);
+ volumeChoosingPolicy = mock(RoundRobinVolumeChoosingPolicy.class);
+ Mockito.when(volumeChoosingPolicy.chooseVolume(anyList(), anyLong()))
+ .thenReturn(hddsVolume);
+
+ for (int i=0; i<2; i++) {
+ KeyValueContainerData keyValueContainerData = new KeyValueContainerData(i,
+ ChunkLayOutVersion.FILE_PER_BLOCK,
+ (long) StorageUnit.GB.toBytes(5), UUID.randomUUID().toString(),
+ datanodeId.toString());
+
+ KeyValueContainer keyValueContainer =
+ new KeyValueContainer(keyValueContainerData,
+ conf);
+ keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId);
+
+
+ List<Long> blkNames;
+ if (i % 2 == 0) {
+ blkNames = addBlocks(keyValueContainer, true);
+ markBlocksForDelete(keyValueContainer, true, blkNames, i);
+ } else {
+ blkNames = addBlocks(keyValueContainer, false);
+ markBlocksForDelete(keyValueContainer, false, blkNames, i);
+ }
+
+ }
+ }
+
+
+ private void markBlocksForDelete(KeyValueContainer keyValueContainer,
+ boolean setMetaData, List<Long> blockNames, int count) throws Exception {
+ try(ReferenceCountedDB metadataStore = BlockUtils.getDB(keyValueContainer
+ .getContainerData(), conf)) {
+
+ for (int i = 0; i < count; i++) {
+ byte[] blkBytes = Longs.toByteArray(blockNames.get(i));
+ byte[] blkInfo = metadataStore.getStore().get(blkBytes);
+
+ byte[] deletingKeyBytes =
+ DFSUtil.string2Bytes(OzoneConsts.DELETING_KEY_PREFIX +
+ blockNames.get(i));
+
+ metadataStore.getStore().delete(blkBytes);
+ metadataStore.getStore().put(deletingKeyBytes, blkInfo);
+ }
+
+ if (setMetaData) {
+ metadataStore.getStore().put(DB_PENDING_DELETE_BLOCK_COUNT_KEY,
+ Longs.toByteArray(count));
+ long blkCount = Longs.fromByteArray(
+ metadataStore.getStore().get(DB_BLOCK_COUNT_KEY));
+ metadataStore.getStore().put(DB_BLOCK_COUNT_KEY,
+ Longs.toByteArray(blkCount - count));
+ long bytesUsed = Longs.fromByteArray(
+ metadataStore.getStore().get(DB_CONTAINER_BYTES_USED_KEY));
+ metadataStore.getStore().put(DB_CONTAINER_BYTES_USED_KEY,
+ Longs.toByteArray(bytesUsed - (count * blockLen)));
+
+ }
+ }
+
+ }
+
+ private List<Long> addBlocks(KeyValueContainer keyValueContainer,
+ boolean setMetaData) throws Exception {
+ long containerId = keyValueContainer.getContainerData().getContainerID();
+
+ List<Long> blkNames = new ArrayList<>();
+ try(ReferenceCountedDB metadataStore = BlockUtils.getDB(keyValueContainer
+ .getContainerData(), conf)) {
+
+ for (int i = 0; i < blockCount; i++) {
+ // Creating BlockData
+ BlockID blockID = new BlockID(containerId, i);
+ BlockData blockData = new BlockData(blockID);
+ blockData.addMetadata(OzoneConsts.VOLUME, OzoneConsts.OZONE);
+ blockData.addMetadata(OzoneConsts.OWNER,
+ OzoneConsts.OZONE_SIMPLE_HDFS_USER);
+ List<ContainerProtos.ChunkInfo> chunkList = new ArrayList<>();
+ ChunkInfo info = new ChunkInfo(String.format("%d.data.%d", blockID
+ .getLocalID(), 0), 0, blockLen);
+ chunkList.add(info.getProtoBufMessage());
+ blockData.setChunks(chunkList);
+ blkNames.add(blockID.getLocalID());
+ metadataStore.getStore().put(Longs.toByteArray(blockID.getLocalID()),
+ blockData
+ .getProtoBufMessage().toByteArray());
+ }
+
+ if (setMetaData) {
+ metadataStore.getStore().put(DB_BLOCK_COUNT_KEY,
+ Longs.toByteArray(blockCount));
+ metadataStore.getStore().put(OzoneConsts.DB_CONTAINER_BYTES_USED_KEY,
+ Longs.toByteArray(blockCount * blockLen));
+ }
+ }
+
+ return blkNames;
+ }
+
+ @Test
+ public void testContainerReader() throws Exception {
+ ContainerReader containerReader = new ContainerReader(volumeSet,
+ hddsVolume, containerSet, conf);
+
+ Thread thread = new Thread(containerReader);
+ thread.start();
+ thread.join();
+
+ Assert.assertEquals(2, containerSet.containerCount());
+
+ for (int i=0; i < 2; i++) {
+ Container keyValueContainer = containerSet.getContainer(i);
+
+ KeyValueContainerData keyValueContainerData = (KeyValueContainerData)
+ keyValueContainer.getContainerData();
+
+ // Verify block related metadata.
+ Assert.assertEquals(blockCount - i,
+ keyValueContainerData.getKeyCount());
+
+ Assert.assertEquals((blockCount - i) * blockLen,
+ keyValueContainerData.getBytesUsed());
+
+ Assert.assertEquals(i,
+ keyValueContainerData.getNumPendingDeletionBlocks());
+ }
+ }
+}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
index efcbb10..77dcb26 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
@@ -29,6 +29,7 @@
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
import org.apache.hadoop.ozone.container.common.impl.ChunkLayOutVersion;
@@ -239,6 +240,12 @@
blockData.getProtoBufMessage().toByteArray());
}
+ // Set Block count and used bytes.
+ db.getStore().put(OzoneConsts.DB_BLOCK_COUNT_KEY,
+ Longs.toByteArray(blocks));
+ db.getStore().put(OzoneConsts.DB_CONTAINER_BYTES_USED_KEY,
+ Longs.toByteArray(usedBytes));
+
// remaining available capacity of the container
return (freeBytes - usedBytes);
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java
index a136983..ecb7af8 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java
@@ -16,18 +16,18 @@
*/
package org.apache.hadoop.ozone.container.testutils;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.container.keyvalue.statemachine.background
- .BlockDeletingService;
-import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
-
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.ozone.container.keyvalue.statemachine.background.BlockDeletingService;
+import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
/**
* A test class implementation for {@link BlockDeletingService}.
*/
@@ -43,7 +43,7 @@
private AtomicInteger numOfProcessed = new AtomicInteger(0);
public BlockDeletingServiceTestImpl(OzoneContainer container,
- int serviceInterval, Configuration conf) {
+ int serviceInterval, ConfigurationSource conf) {
super(container, serviceInterval, SERVICE_TIMEOUT_IN_MILLISECONDS,
TimeUnit.MILLISECONDS, conf);
}
diff --git a/hadoop-hdds/dev-support/checkstyle/checkstyle.xml b/hadoop-hdds/dev-support/checkstyle/checkstyle.xml
index 1c43741..c4ebfb6 100644
--- a/hadoop-hdds/dev-support/checkstyle/checkstyle.xml
+++ b/hadoop-hdds/dev-support/checkstyle/checkstyle.xml
@@ -49,6 +49,10 @@
<module name="Checker">
+ <module name="BeforeExecutionExclusionFileFilter">
+ <property name="fileNamePattern" value=".*/target/generated.*"/>
+ </module>
+
<module name="SuppressWarningsFilter"/>
<!-- Checks that a package.html file exists for each package. -->
@@ -115,7 +119,10 @@
<!-- Checks for imports -->
<!-- See http://checkstyle.sf.net/config_import.html -->
- <module name="IllegalImport"/> <!-- defaults to sun.* packages -->
+ <module name="IllegalImport">
+ <property name="regexp" value="true"/>
+ <property name="illegalPkgs" value="^sun\..*, ^.*\.relocated\..*, ^.*\.shaded\..*"/>
+ </module>
<module name="RedundantImport"/>
<module name="UnusedImports"/>
@@ -192,5 +199,4 @@
<module name="UpperEll"/>
</module>
-
</module>
diff --git a/hadoop-hdds/docs/archetypes/design.md b/hadoop-hdds/docs/archetypes/design.md
new file mode 100644
index 0000000..59dc5f2
--- /dev/null
+++ b/hadoop-hdds/docs/archetypes/design.md
@@ -0,0 +1,60 @@
+---
+title: "{{ replace .Name "-" " " | title }}"
+menu: main
+jira: HDDS-XXX
+summary: One sentence summary. Will be displayed at the main design doc table.
+status: current
+author: Your Name
+
+---
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+
+This the proposed template to document any proposal. It's recommended but not required the use exactly the some structure. Some proposal may require different structure, but we need the following information.
+
+## Summary
+
+> Give a one sentence summary, like the jira title. It will be displayed on the documentation page. Should be enough to understand
+
+## Problem statement (Motivation / Abstract)
+
+> What is the problem and how would you solve it? Think about an abstract of a paper: one paragraph overview. Why will the world better with this change?
+
+## Non-goals
+
+ > Very important to define what is outside of the scope of this proposal
+
+## Technical Description (Architecture and implementation details)
+
+ > Explain the problem in more details. How can it be reproduced? What is the current solution? What is the limitation of the current solution?
+
+ > How the new proposed solution would solve the problem? Architectural design.
+
+ > Implementation details. What should be changed in the code. Is it a huge change? Do we need to change wire protocol? Backward compatibility?
+
+## Alternatives
+
+ > What are the other alternatives you considered and why do yoy prefer the proposed solution The goal of this section is to help people understand why this is the best solution now, and also to prevent churn in the future when old alternatives are reconsidered.
+
+Note: In some cases 4/5 can be combined. For example if you have multiple proposals, the first version may include multiple solutions. At the end ot the discussion we can move the alternatives to 5. and explain why the community is decided to use the selected option.
+
+## Implementation plan
+
+ > Planning to implement the feature. Estimated size of the work? Do we need feature branch? Any migration plan, dependency? If it's not a big new feature it can be one sentence or optiona;.
+
+## References
\ No newline at end of file
diff --git a/hadoop-hdds/docs/content/beyond/Containers.zh.md b/hadoop-hdds/docs/content/beyond/Containers.zh.md
new file mode 100644
index 0000000..c06902e
--- /dev/null
+++ b/hadoop-hdds/docs/content/beyond/Containers.zh.md
@@ -0,0 +1,203 @@
+---
+title: "Ozone 中的容器技术"
+summary: Ozone 广泛地使用容器来进行测试，本页介绍 Ozone 中容器的使用及其最佳实践。
+weight: 2
+---
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+Ozone 的开发中大量地使用了 Docker，包括以下三种主要的应用场景：
+
+* __开发__:
+ * 我们使用 docker 来启动本地伪集群（docker 可以提供统一的环境，但是不需要创建镜像）。
+* __测试__:
+ * 我们从开发分支创建 docker 镜像，然后在 kubernetes 或其它容器编排系统上测试 ozone。
+ * 我们为每个发行版提供了 _apache/ozone_ 镜像，以方便用户体验 Ozone。
+ 这些镜像 __不__ 应当在 __生产__ 中使用。
+
+<div class="alert alert-warning" role="alert">
+当在生产中使用容器方式部署 ozone 时，我们<b>强烈</b>建议你创建自己的镜像。请把所有自带的容器镜像和 k8s 资源文件当作示例指南，参考它们进行定制。
+</div>
+
+* __生产__:
+ * 我们提供了如何为生产集群创建 docker 镜像的文档。
+
+下面我们来详细地介绍一下各种应用场景：
+
+## 开发
+
+Ozone 安装包中包含了 docker-compose 的示例目录，用于方便地在本地机器启动 Ozone 集群。
+
+使用官方提供的发行包：
+
+```bash
+cd compose/ozone
+docker-compose up -d
+```
+
+本地构建方式：
+
+```bash
+cd hadoop-ozone/dist/target/ozone-*/compose
+docker-compose up -d
+```
+
+这些 compose 环境文件是重要的工具，可以用来随时启动各种类型的 Ozone 集群。
+
+为了确保 compose 文件是最新的，我们提供了验收测试套件，套件会启动集群并检查其基本行为是否正常。
+
+验收测试也包含在发行包中，你可以在 `smoketest` 目录下找到各个测试的定义。
+
+你可以在任意 compose 目录进行测试，比如：
+
+```bash
+cd compose/ozone
+./test.sh
+```
+
+### 实现细节
+
+`compose` 测试都基于 apache/hadoop-runner 镜像，这个镜像本身并不包含任何 Ozone 的 jar 包或二进制文件，它只是提供其了启动 Ozone 的辅助脚本。
+
+hadoop-runner 提供了一个随处运行 Ozone 的固定环境，Ozone 分发包通过目录挂载包含在其中。
+
+(docker-compose 示例片段)
+
+```
+ scm:
+ image: apache/hadoop-runner:jdk11
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9876:9876
+
+```
+
+容器应该通过环境变量来进行配置，由于每个容器都应当设置相同的环境变量，我们在单独的文件中维护了一个环境变量列表：
+
+```
+ scm:
+ image: apache/hadoop-runner:jdk11
+ #...
+ env_file:
+ - ./docker-config
+```
+
+docker-config 文件中包含了所需环境变量的列表：
+
+```
+OZONE-SITE.XML_ozone.om.address=om
+OZONE-SITE.XML_ozone.om.http-address=om:9874
+OZONE-SITE.XML_ozone.scm.names=scm
+#...
+```
+
+你可以看到我们所使用的命名规范，根据这些环境变量的名字，`hadoop-runner` 基础镜像中的[脚本](https://github.com/apache/hadoop/tree/docker-hadoop-runner-latest/scripts) 会生成合适的 hadoop XML 配置文件（在我们这种情况下就是 `ozone-site.xml`）。
+
+`hadoop-runner` 镜像的[入口点](https://github.com/apache/hadoop/blob/docker-hadoop-runner-latest/scripts/starter
+.sh)包含了一个辅助脚本，这个辅助脚本可以根据环境变量触发上述的配置文件生成以及其它动作（比如初始化 SCM 和 OM 的存储、下载必要的 keytab 等）。
+
+## 测试
+
+`docker-compose` 的方式应当只用于本地测试，不适用于多节点集群。要在多节点集群上使用容器，我们需要像 Kubernetes 这样的容器编排系统。
+
+Kubernetes 示例文件在 `kubernetes` 文件夹中。
+
+*请注意*：所有提供的镜像都使用 `hadoop-runner` 作为基础镜像，这个镜像中包含了所有测试环境所需的测试工具。对于生产环境，我们推荐用户使用自己的基础镜像创建可靠的镜像。
+
+### 发行包测试
+
+可以通过部署任意的示例集群来测试发行包：
+
+```bash
+cd kubernetes/examples/ozone
+kubectl apply -f
+```
+
+注意，在这个例子中会从 Docker Hub 下载最新的镜像。
+
+### 开发构建测试
+
+为了测试开发中的构建，你需要创建自己的镜像并上传到自己的 docker 仓库中：
+
+
+```bash
+mvn clean install -DskipTests -Pdocker-build,docker-push -Ddocker.image=myregistry:9000/name/ozone
+```
+
+所有生成的 kubernetes 资源文件都会使用这个镜像 (`image:` keys are adjusted during the build)
+
+```bash
+cd kubernetes/examples/ozone
+kubectl apply -f
+```
+
+## 生产
+
+<div class="alert alert-danger" role="alert">
+我们<b>强烈</b>推荐在生产集群使用自己的镜像，并根据实际的需求调整基础镜像、文件掩码、安全设置和用户设置。
+</div>
+
+你可以使用我们开发中所用的镜像作为示例：
+
+ * [基础镜像] (https://github.com/apache/hadoop/blob/docker-hadoop-runner-jdk11/Dockerfile)
+ * [完整镜像] (https://github.com/apache/hadoop/blob/trunk/hadoop-ozone/dist/src/main/docker/Dockerfile)
+
+ Dockerfile 中大部分内容都是可选的辅助功能，但如果要使用我们提供的 kubernetes 示例资源文件，你可能需要[这里](https://github.com/apache/hadoop/tree/docker-hadoop-runner-jdk11/scripts)的脚本。
+
+ * 两个 python 脚本将环境变量转化为实际的 hadoop XML 配置文件
+ * start.sh 根据环境变量执行 python 脚本（以及其它初始化工作）
+
+## 容器
+
+Ozone 相关的容器镜像和 Dockerfile 位置：
+
+
+<table class="table table-dark">
+ <thead>
+ <tr>
+ <th scope="col">#</th>
+ <th scope="col">容器</th>
+ <th scope="col">仓库</th>
+ <th scope="col">基础镜像</th>
+ <th scope="col">分支</th>
+ <th scope="col">标签</th>
+ <th scope="col">说明</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <th scope="row">1</th>
+ <td>apache/ozone</td>
+ <td>https://github.com/apache/hadoop-docker-ozone</td>
+ <td>ozone-... </td>
+ <td>hadoop-runner</td>
+ <td>0.3.0,0.4.0,0.4.1</td>
+ <td>每个 Ozone 发行版都对应一个新标签。</td>
+ </tr>
+ <tr>
+ <th scope="row">2</th>
+ <td>apache/hadoop-runner </td>
+ <td>https://github.com/apache/hadoop</td>
+ <td>docker-hadoop-runner</td>
+ <td>centos</td>
+ <td>jdk11,jdk8,latest</td>
+ <td>这是用于测试 Hadoop Ozone 的基础镜像，包含了一系列可以让我们更加方便地运行 Ozone 的工具。
+ </td>
+ </tr>
+ </tbody>
+</table>
diff --git a/hadoop-hdds/docs/content/beyond/_index.zh.md b/hadoop-hdds/docs/content/beyond/_index.zh.md
new file mode 100644
index 0000000..b7f6775
--- /dev/null
+++ b/hadoop-hdds/docs/content/beyond/_index.zh.md
@@ -0,0 +1,27 @@
+---
+title: "进阶"
+date: "2017-10-10"
+menu: main
+weight: 7
+
+---
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+{{<jumbotron title="进阶">}}
+ 本部分介绍 Ozone 的自定义配置，包括如何将 Ozone 以并存的方式部署到已有的 HDFS 集群，以及如何运行 Ozone 内置的 profilers 和 tracing 功能。
+{{</jumbotron>}}
diff --git a/hadoop-hdds/docs/content/concept/Datanodes.md b/hadoop-hdds/docs/content/concept/Datanodes.md
index ea63fe4..de07058 100644
--- a/hadoop-hdds/docs/content/concept/Datanodes.md
+++ b/hadoop-hdds/docs/content/concept/Datanodes.md
@@ -71,5 +71,5 @@
blocks that get reported. That is a 40x reduction in the block reports.
This extra indirection helps tremendously with scaling Ozone. SCM has far
-less block data to process and the name node is a different service are
-critical to scaling Ozone.
+less block data to process and the namespace service (Ozone Manager) as a
+different service are critical to scaling Ozone.
diff --git a/hadoop-hdds/docs/content/concept/Datanodes.zh.md b/hadoop-hdds/docs/content/concept/Datanodes.zh.md
index 1c785a1..fa992dc 100644
--- a/hadoop-hdds/docs/content/concept/Datanodes.zh.md
+++ b/hadoop-hdds/docs/content/concept/Datanodes.zh.md
@@ -1,8 +1,8 @@
---
-title: "Datanodes"
+title: "数据节点"
date: "2017-09-14"
weight: 4
-summary: TODO translated summary
+summary: Ozone 支持 Amazon S3 协议，你可以原封不动地在 Ozone 上使用基于 S3 客户端和 S3 SDK 的应用。
---
<!---
Licensed to the Apache Software Foundation (ASF) under one or more
@@ -21,4 +21,28 @@
limitations under the License.
-->
-TODO: content translations
\ No newline at end of file
+数据节点是 Ozone 中的 worker，所有的数据都存储在数据节点上，用户以块的方式写数据，数据节点将多个块聚合成一个存储容器，存储容器中包含用户写入的数据块和这些块的元数据。
+
+## 存储容器
+
+
+
+Ozone 的存储容器是一个自包含的超级块，容器中包含一系列的 Ozone 块，以及存储实际数据的磁盘文件，这是默认的存储容器格式。对于 Ozone 来说，容器只是提供了一个协议规范，它独立于具体的存储格式实现，换句话说，我们可以很容易扩展或引入新的容器实现格式。因此，上述格式应当被看作是 Ozone 存储容器的参考实现。
+
+## 理解 Ozone 中的块和容器
+
+当用户想要从 Ozone 中读取一个键时，用户向 OM 发送该键名，OM 会返回组成该键的块列表。
+
+每个 Ozone 块包含一个容器 ID 和一个本地 ID，下图展示了 Ozone 块的逻辑组成：
+
+
+
+容器 ID 用来让用户发现容器的位置，容器位置的权威信息存储在 SCM 中。大部分情况下，OM 会缓存容器的位置信息，并会将它们随 Ozone 块一起返回给用户。
+
+当用户定位到容器的位置，即知道哪些数据节点包含这个容器后，用户会连接数据节点，然后读取由 _容器ID:本地ID_ 指定的数据流，换句话说，本地 ID 相当于容器内的索引，描述了我们应该读取哪个数据流。
+
+### 容器的位置发现
+
+SCM 如何获得容器的位置？这一点和现有的 HDFS 十分相似。数据节点会定期发送类似于块报告的容器报告，容器报告比块报告的内容简洁的多，比如，对于一个存储容量为 196 TB 的集群，Ozone 大概会拥有四万个容器，相比于 HDFS 的一百五十万个块，块报告数量缩减为四十分之一。
+
+这种间接管理的方式大大地提高了 Ozone 的扩展性，因为 SCM 需要处理的块数据大大减少，且命名服务（OM）作为一个独特的服务主体对于扩展 Ozone 具有重要意义。
diff --git a/hadoop-hdds/docs/content/design/configless.md b/hadoop-hdds/docs/content/design/configless.md
new file mode 100644
index 0000000..bea8b39
--- /dev/null
+++ b/hadoop-hdds/docs/content/design/configless.md
@@ -0,0 +1,31 @@
+---
+title: Configless Ozone service management
+summary: Distribute only minimal configuration and download all the remaining before start
+date: 2019-05-25
+jira: HDDS-1467
+status: accepted
+author: Márton Elek, Anu Engineer
+---
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Abstract
+
+ Configuration keys are partitioned to two parts: runtime settings and environment settings.
+
+ Environment settings (hosts, ports) can be downloaded at start.
+
+# Link
+
+ * https://issues.apache.org/jira/secure/attachment/12966992/configless.pdf
\ No newline at end of file
diff --git a/hadoop-hdds/docs/content/design/decommissioning.md b/hadoop-hdds/docs/content/design/decommissioning.md
index 8d620be..6c8e08e 100644
--- a/hadoop-hdds/docs/content/design/decommissioning.md
+++ b/hadoop-hdds/docs/content/design/decommissioning.md
@@ -1,3 +1,11 @@
+---
+title: Decommissioning in Ozone
+summary: Formal process to shut down machines in a safe way after the required replications.
+date: 2019-07-31
+jira: HDDS-1881
+status: implementing
+author: Anu Engineer, Marton Elek, Stephen O'Donnell
+---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -12,15 +20,6 @@
limitations under the License. See accompanying LICENSE file.
-->
----
-title: Decommissioning in Ozone
-summary: Formal process to shut down machines in a safe way after the required replications.
-date: 2019-07-31
-jira: HDDS-1881
-status: current
-author: Anu Engineer, Marton Elek, Stephen O'Donnell
----
-
# Abstract
The goal of decommissioning is to turn off a selected set of machines without data loss. It may or may not require to move the existing replicas of the containers to other nodes.
diff --git a/hadoop-hdds/docs/content/design/gdpr.md b/hadoop-hdds/docs/content/design/gdpr.md
new file mode 100644
index 0000000..a0b06a8
--- /dev/null
+++ b/hadoop-hdds/docs/content/design/gdpr.md
@@ -0,0 +1,31 @@
+---
+title: Ozone GDPR framework
+summary: Crypto key management to handle GDPR "right to be forgotten" feature
+date: 2019-06-20
+jira: HDDS-2012
+status: implemented
+author: Dinesh Chitlangia
+---
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Abstract
+
+ Encrypt all the key and delete only the encryption key if the deletion is requested. Without the key any remaining data can't be read any more.
+
+# Link
+
+ https://issues.apache.org/jira/secure/attachment/12978992/Ozone%20GDPR%20Framework.pdf
+
+ https://issues.apache.org/jira/secure/attachment/12987528/Ozone%20GDPR%20Framework_updated.pdf
\ No newline at end of file
diff --git a/hadoop-hdds/docs/content/design/locks.md b/hadoop-hdds/docs/content/design/locks.md
new file mode 100644
index 0000000..79ae8d8
--- /dev/null
+++ b/hadoop-hdds/docs/content/design/locks.md
@@ -0,0 +1,29 @@
+---
+title: Ozone locks in OM
+summary: Hierarchical locking structure for OM elements (volumes/keys/buckets)
+date: 2019-06-20
+jira: HDDS-1672
+status: implemented
+author: Bharat Viswanadham
+---
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Abstract
+
+ Fine grained locking hierarchy for Ozone specific resources (volumes, buckets, keys...)
+
+# Link
+
+ https://issues.apache.org/jira/secure/attachment/12972275/Ozone%20Locks%20in%20OM.pdf
\ No newline at end of file
diff --git a/hadoop-hdds/docs/content/design/multiraft.md b/hadoop-hdds/docs/content/design/multiraft.md
new file mode 100644
index 0000000..bccaff3
--- /dev/null
+++ b/hadoop-hdds/docs/content/design/multiraft.md
@@ -0,0 +1,31 @@
+---
+title: Ozone multi-raft support
+summary: Datanodes can be part of multiple independent RAFT groups / pipelines
+date: 2019-05-21
+jira: HDDS-1564
+status: implemented
+author:
+---
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Abstract
+
+ Without multiraft support one datanode could be part of one RAFT group. With multiraft one datanode can be part of multiple independent RAFT ring.
+
+# Link
+
+ https://issues.apache.org/jira/secure/attachment/12990694/multiraft_performance_brief.pdf
+
+ https://issues.apache.org/jira/secure/attachment/12969227/Ozone%20Multi-Raft%20Support.pdf
\ No newline at end of file
diff --git a/hadoop-hdds/docs/content/design/namespace-support.md b/hadoop-hdds/docs/content/design/namespace-support.md
new file mode 100644
index 0000000..0317b46
--- /dev/null
+++ b/hadoop-hdds/docs/content/design/namespace-support.md
@@ -0,0 +1,29 @@
+---
+title: Ozone Manager HA
+summary: Support HA for Ozone Manager with the help of RATIS
+date: 2020-01-20
+jira: HDDS-2939
+status: accepted
+author: Supratim Deka, Anu Engineer
+---
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Abstract
+
+ Flat namespace (like key -> key info) is not efficient for retrieveing directories. (Large segments should be scanned, the whole sub-hierarchy)
+
+# Link
+
+ * https://issues.apache.org/jira/secure/attachment/12991926/Ozone%20FS%20Namespace%20Proposal%20v1.0.docx
diff --git a/hadoop-hdds/docs/content/design/nfs.md b/hadoop-hdds/docs/content/design/nfs.md
new file mode 100644
index 0000000..05ad827
--- /dev/null
+++ b/hadoop-hdds/docs/content/design/nfs.md
@@ -0,0 +1,31 @@
+---
+title: NFS support Ozone
+summary: Adopt NFS gateway of HDFS and provide NFS file system view
+date: 2020-03-25
+jira: HDDS-3001
+status: draft
+author: Prashant Pogde
+---
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Abstract
+
+ 1. Use NFS server of HDFS and adopt it
+ 2. Support unique ID for each Ozone object
+ 3. Support Random Write
+
+# Link
+
+ * https://issues.apache.org/jira/secure/attachment/12997790/NFS%20Support%20for%20Ozone.pdf
diff --git a/hadoop-hdds/docs/content/design/ofs.md b/hadoop-hdds/docs/content/design/ofs.md
new file mode 100644
index 0000000..71e9617
--- /dev/null
+++ b/hadoop-hdds/docs/content/design/ofs.md
@@ -0,0 +1,33 @@
+---
+title: Implement new Ozone FileSystem scheme ofs://
+summary: A new schema structure for Hadoop compatible file system
+date: 2019-12-05
+jira: HDDS-2665
+status: implementing
+author: Siyao Meng
+---
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Abstract
+
+ Existing scheme: o3fs://bucket.volume/key/../...
+
+ Proposed scheme: ofs://omhost/volume/bucket/key/../...
+
+# Link
+
+Design doc is uploaded to the JIRA:
+
+https://issues.apache.org/jira/secure/attachment/12987636/Design%20ofs%20v1.pdf
\ No newline at end of file
diff --git a/hadoop-hdds/docs/content/design/omha.md b/hadoop-hdds/docs/content/design/omha.md
new file mode 100644
index 0000000..652144f
--- /dev/null
+++ b/hadoop-hdds/docs/content/design/omha.md
@@ -0,0 +1,33 @@
+---
+title: Ozone Manager HA
+summary: Support HA for Ozone Manager with the help of RATIS
+date: 2018-09-18
+jira: HDDS-505
+status: implemented
+author: Bharat Viswanadham
+---
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Abstract
+
+ HA for Ozone Manager with the help of Ratis. High performance operation with caching and double-buffer (separated doc).
+
+# Link
+
+ * https://issues.apache.org/jira/secure/attachment/12940314/OzoneManager%20HA.pdf
+
+ * https://issues.apache.org/jira/secure/attachment/12990063/OM%20HA%20Cache%20Design.pdf
+
+ * https://issues.apache.org/jira/secure/attachment/12973260/Handling%20Write%20Requests%20with%20OM%20HA.pdf
\ No newline at end of file
diff --git a/hadoop-hdds/docs/content/design/ozone-enhancement-proposals.md b/hadoop-hdds/docs/content/design/ozone-enhancement-proposals.md
index cc7569e..b795cd7 100644
--- a/hadoop-hdds/docs/content/design/ozone-enhancement-proposals.md
+++ b/hadoop-hdds/docs/content/design/ozone-enhancement-proposals.md
@@ -129,7 +129,7 @@
Defined in the markdown header. Proposed statuses:
- * `accepted`: (Use this as by default. If not accapted, won't be merged)
+ * `accepted`: (Use this as by default. If not accepted, won't be merged)
* `implemented`: The discussed technical solution is implemented (maybe with some minor implementation difference)
diff --git a/hadoop-hdds/docs/content/design/ozone-volume-management.md b/hadoop-hdds/docs/content/design/ozone-volume-management.md
new file mode 100644
index 0000000..6c63656
--- /dev/null
+++ b/hadoop-hdds/docs/content/design/ozone-volume-management.md
@@ -0,0 +1,181 @@
+---
+title: Ozone Volume Management
+summary: A simplified version of mapping between S3 buckets and Ozone volume/buckets
+date: 2020-04-02
+jira: HDDS-3331
+status: accepted
+author: Marton Elek, Arpit Agarwall, Sunjay Radia
+---
+
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+
+## Introduction
+
+This document explores how we can improve the Ozone volume semantics especially with respect to the S3 compatibility layer.
+
+## The Problems
+
+ 1. Unpriviliged users cannot enumerate volumes.
+ 2. The mapping of S3 buckets to Ozone volumes is confusing. Based on external feedback it's hard to understand the exact Ozone URL to be used.
+ 3. The volume name is not friendly and cannot be remembered by humans.
+ 4. Ozone buckets created via the native object store interface are not visible via the S3 gateway.
+ 5. We don't support the revocation of access keys.
+
+We explore some of these in more detail in subsequent sections.
+
+### Volume enumeration problem
+
+Currently when a user enumerates volumes, they see the list of volumes that they own. This means that when an unprivileged user enumerates volumes, it always gets any empty list. Instead users should be able to see all volumes that they have been granted read or write access to.
+
+This also has an impact on [ofs](https://issues.apache.org/jira/browse/HDDS-2665) which makes volumes appear as top-level directories.
+
+### S3 to HCFS path mapping problem
+
+Ozone has the semantics of volume *and* buckets while S3 has only buckets. To make it possible to use the same bucket both from Hadoop world and via S3 we need a mapping between them.
+
+Currently we maintain a map between the S3 buckets and Ozone volumes + buckets in `OmMetadataManagerImpl`
+
+```
+s3_bucket --> ozone_volume/ozone_bucket
+```
+
+The current implementation uses the `"s3" + s3UserName` string as the volume name and the `s3BucketName` as the bucket name. Where `s3UserName` is is the `DigestUtils.md5Hex(kerberosUsername.toLowerCase())`
+
+To create an S3 bucket and use it from o3fs, you should:
+
+1. Get your personal secret based on your kerberos keytab
+
+```
+> kinit -kt /etc/security/keytabs/testuser.keytab testuser/scm
+> ozone s3 getsecret
+awsAccessKey=testuser/scm@EXAMPLE.COM
+awsSecret=7a6d81dbae019085585513757b1e5332289bdbffa849126bcb7c20f2d9852092
+```
+
+2. Create the bucket with S3 cli
+
+```
+> export AWS_ACCESS_KEY_ID=testuser/scm@EXAMPLE.COM
+> export AWS_SECRET_ACCESS_KEY=7a6d81dbae019085585513757b1e5332289bdbffa849126bcb7c20f2d9852092
+> aws s3api --endpoint http://localhost:9878 create-bucket --bucket=bucket1
+```
+
+3. And identify the ozone path
+
+```
+> ozone s3 path bucket1
+Volume name for S3Bucket is : s3c89e813c80ffcea9543004d57b2a1239
+Ozone FileSystem Uri is : o3fs://bucket1.s3c89e813c80ffcea9543004d57b2a1239
+```
+
+## Proposed solution[1]
+
+### Supporting multiple access keys (#5 from the problem listing)
+
+Problem #5 can be easily supported with improving the `ozone s3` CLI. Ozone has a separated table for the S3 secrets and the API can be improved to handle multiple secrets for one specific kerberos user.
+
+### Solving the mapping problem (#2-4 from the problem listing)
+
+ 1. Let's always use `s3` volume for all the s3 buckets **if the bucket is created from the s3 interface**.
+
+This is an easy an fast method, but with this approach not all the volumes are avilable via the S3 interface. We need to provide a method to publish any of the ozone volumes / buckets.
+
+ 2. Let's improve the existing toolset to expose **any** Ozone volume/bucket as an s3 bucket. (Eg. expose `o3:/vol1/bucketx` as an S3 bucket `s3://foobar` )
+
+**Implementation**:
+
+ The first part is easy compared to the current implementation. We don't need any mapping table any more.
+
+ To implement the second (expose ozone buckets as s3 buckets) we have multiple options:
+
+ 1. Store some metadata (** s3 bucket name **) on each of the buckets
+ 2. Implement a **bind mount** mechanic which makes it possible to *mount* any volume/buckets to the specific "s3" volume.
+
+The first approach required a secondary cache table and it violates the naming hierarchy. The s3 bucket name is a global unique name, therefore it's more than just a single attribute on a specific object. It's more like an element in the hierachy. For this reason the second option is proposed:
+
+For example if the default s3 volume is `s3`
+
+ 1. Every new buckets created via s3 interface will be placed under the `/s3` volume
+ 2. Any existing **Ozone** buckets can be exposed with mounting it to s3: `ozone sh mount /vol1/bucket1 /s3/s3bucketname`
+
+**Lock contention problem**
+
+One possible problem with using just one volume is using the locks of the same volume for all the D3 buckets (thanks Xiaoyu). But this shouldn't be a big problem.
+
+ 1. We hold only a READ lock. Most of the time it can acquired without any contention (writing lock is required only to change owner / set quota)
+ 2. For symbolic link / bind mounts the read lock is only required for the first read. After that the lock of the referenced volume will be used. In case of any performance problem multiple volumes + bind mounts can be used.
+
+Note: Sunjay is added to the authors as the original proposal of this approach.
+
+## Alternative approaches and reasons to reject
+
+To solve the the _s3 bucket name to ozone bucket name mapping_ problem some other approaches are also considered. They are rejected but keeping them in this section together with the reasons to reject.
+
+
+### 1. Predefined volume mapping
+
+ 1. Let's support multiple `ACCESS_KEY_ID` for the same user.
+ 2. For each `ACCESS_KEY_ID` a volume name MUST be defined.
+ 3. Instead of using a specific mapping table, the `ACCESS_KEY_ID` would provide a **view** of the buckets in the specified volume.
+
+With this approach the used volume will be more visible and -- hopefully -- understandable.
+
+Instead of using `ozone s3 getsecret`, following commands would be used:
+
+ 1. `ozone s3 secret create --volume=myvolume`: To create a secret and use myvolume for all of these buckets
+ 2. `ozone s3 secret list`: To list all of the existing S3 secrets (available for the current user)
+ 3. `ozone s3 secret delete <ACCESS_KEY_ID`: To delete any secret
+
+The `AWS_ACCESS_KEY_ID` should be a random identifier instead of using a kerberos principal.
+
+ * __pro__: Easier to understand
+ * __con__: We should either have global unique bucket names or it will be possible to see two different buckets with
+ * __con__: It can be hard to remember which volumes are assigned to a specific ACCESS_KEY_ID
+
+### 3. String Magic
+
+We can try to make volume name visible for the S3 world by using some structured bucket names. Unfortunately the available separator characters are very limited:
+
+For example we can't use `/`
+
+```
+aws s3api create-bucket --bucket=vol1/bucket1
+
+Parameter validation failed:
+Invalid bucket name "vol1/bucket1": Bucket name must match the regex "^[a-zA-Z0-9.\-_]{1,255}$" or be an ARN matching the regex "^arn:(aws).*:s3:[a-z\-0-9]+:[0-9]{12}:accesspoint[/:][a-zA-Z0-9\-]{1,63}$"
+```
+
+But it's possible to use `volume-bucket` notion:
+
+```
+aws s3api create-bucket --bucket=vol1-bucket1
+```
+ * __pro__: Volume mapping is visible all the time.
+ * __con__: Harder to use any external tool with defaults (all the buckets should have at least one `-`)
+ * __con__: Hierarchy is not visble. The uniform way to separated elements in fs hierarchy is `/`. It can be confusing.
+ *
+### 4. Remove volume From OzoneFs Paths
+
+We can also make volumes a lightweight *bucket group* object by removing it from the ozonefs path. With this approach we can use all the benefits of the volumes as an administration object but it would be removed from the `o3fs` path.
+
+ * __pro__: can be the most simple solution. Easy to understand as there are no more volumes in the path.
+ * __con__: Bigger change (all the API can't be modified to make volumes optional)
+ * __con__: Harder to dis-joint namespaces based on volumes. (With the current scheme, it's easier to delegate the responsibilties for one volumes to a different OM).
+ * __con__: We lose volumes as the top-level directories in `ofs` scheme.
+ * __con__: One level of hierarchy might not be enough in case of multi-tenancy.
+ * __con__: One level of hierarchy is not enough if we would like to provide separated level for users and admins
+ * __con__: Hierarchical abstraction can be easier to manage and understand
+
diff --git a/hadoop-hdds/docs/content/design/recon1.md b/hadoop-hdds/docs/content/design/recon1.md
new file mode 100644
index 0000000..ed4ff25
--- /dev/null
+++ b/hadoop-hdds/docs/content/design/recon1.md
@@ -0,0 +1,32 @@
+---
+title: Recon server 1
+summary: Second phase of Recon development. Recon acts as a passive SCM.
+date: 2019-02-19
+jira: HDDS-1084
+status: implemented
+author: Sid Wagle, Vivek Subramanian and Aravindan
+---
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Abstract
+
+ Recon is a monitoring and analytics server for Ozone. It can retrieve data from
+ other components and store it and do historical queries (eg. out-of-disk-space time estimation).
+
+> Ozone Recon (short for reconnaissance) Service is, as the name suggests, meant to be eyes and ears of Ozone for the user. The first resort in understanding what is happening with Ozone.
+
+# Link
+
+ * https://issues.apache.org/jira/secure/attachment/12960258/Ozone_Recon_Design_V1_Draft.pdf
diff --git a/hadoop-hdds/docs/content/design/recon2.md b/hadoop-hdds/docs/content/design/recon2.md
new file mode 100644
index 0000000..3d9056b
--- /dev/null
+++ b/hadoop-hdds/docs/content/design/recon2.md
@@ -0,0 +1,29 @@
+---
+title: Recon server v2
+summary: Second phase of Recon development. Recon acts as a passive SCM.
+date: 2019-08-19
+jira: HDDS-1996
+status: implementing
+author: Aravindan Vijayan, Vivek Ratnavel Subramanian, Sid Wagle
+---
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Abstract
+
+ The second phase of Recon development. The biggest change is that Recon can act as a passive SCM, collect HB from Datanode and analyze the data.
+
+# Link
+
+ * https://issues.apache.org/jira/secure/attachment/12986596/Recon%20v0.2.pdf
diff --git a/hadoop-hdds/docs/content/design/s3gateway.md b/hadoop-hdds/docs/content/design/s3gateway.md
new file mode 100644
index 0000000..eb641a6
--- /dev/null
+++ b/hadoop-hdds/docs/content/design/s3gateway.md
@@ -0,0 +1,31 @@
+---
+title: S3 protocol support for Ozone
+summary: Support any AWS S3 compatible client with dedicated REST endpoint
+date: 2018-09-27
+jira: HDDS-434
+status: implemented
+author: Marton Elek, Jitendra Pandey, Bharat Viswanadham & Anu Engineer
+---
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Abstract
+
+Introduction of a stateless separated daemon which acts as an S3 endpoint and transform calls to Ozone RPC calls.
+
+This document is still valid but since the first implementation, more and more features are added.
+
+# Link
+
+ * https://issues.apache.org/jira/secure/attachment/12941573/S3Gateway.pdf
diff --git a/hadoop-hdds/docs/content/design/scmha.md b/hadoop-hdds/docs/content/design/scmha.md
new file mode 100644
index 0000000..a872017
--- /dev/null
+++ b/hadoop-hdds/docs/content/design/scmha.md
@@ -0,0 +1,29 @@
+---
+title: SCM HA support
+summary: HA for Storage Container Manager using Ratis to replicate data
+date: 2020-03-05
+jira: HDDS-2823
+status: draft
+author: Li Cheng
+---
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Abstract
+
+ Proposal to implement HA similar to the OM HA: Using Apache Ratis to propagate the
+
+# Link
+
+ * https://docs.google.com/document/d/1vr_z6mQgtS1dtI0nANoJlzvF1oLV-AtnNJnxAgg69rM/edit?usp=sharing
diff --git a/hadoop-hdds/docs/content/design/tde.md b/hadoop-hdds/docs/content/design/tde.md
new file mode 100644
index 0000000..6bb96c9
--- /dev/null
+++ b/hadoop-hdds/docs/content/design/tde.md
@@ -0,0 +1,29 @@
+---
+title: Ozone Encryption At Rest
+summary: Transparent Data Encryption encrypts the data before store it
+date: 2019-02-08
+jira: HDDS-1041
+status: implemented
+author: Xiaoyu Yao
+---
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Abstract
+
+Design is very similar to HDFS TDE feature. Using KMS encrypts the data before store and decrypt it after reading.
+
+# Link
+
+ https://issues.apache.org/jira/secure/attachment/12957995/Ozone%20Encryption%20At-Rest%20-%20V2019.2.7.pdf
diff --git a/hadoop-hdds/docs/content/design/token.md b/hadoop-hdds/docs/content/design/token.md
new file mode 100644
index 0000000..17bc277
--- /dev/null
+++ b/hadoop-hdds/docs/content/design/token.md
@@ -0,0 +1,25 @@
+---
+title: Generic Extensible Token support for Ozone
+summary: Extend existing token architecture to all entities of Ozone
+date: 2020-01-09
+jira: HDDS-2867
+status: accepted
+author: Xiaoyu Yao
+---
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Link
+
+https://issues.apache.org/jira/secure/attachment/12990459/Generic%20Extensible%20Tokens%20for%20Ozone.pdf
\ No newline at end of file
diff --git a/hadoop-hdds/docs/content/design/trash.md b/hadoop-hdds/docs/content/design/trash.md
new file mode 100644
index 0000000..78e077a
--- /dev/null
+++ b/hadoop-hdds/docs/content/design/trash.md
@@ -0,0 +1,25 @@
+---
+title: Ozone Trash Feature
+summary: Feature provides a user with the ability to recover keys that may have been deleted accidentally. (similar to the HDFS trash feature).
+date: 2019-11-07
+jira: HDDS-2416
+status: implementing
+author: Matthew Sharp
+---
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+The design doc is uploaded to the JIRA:
+
+https://issues.apache.org/jira/secure/attachment/12985273/Ozone_Trash_Feature.docx
\ No newline at end of file
diff --git a/hadoop-hdds/docs/content/design/typesafeconfig.md b/hadoop-hdds/docs/content/design/typesafeconfig.md
new file mode 100644
index 0000000..77a3b4d
--- /dev/null
+++ b/hadoop-hdds/docs/content/design/typesafeconfig.md
@@ -0,0 +1,33 @@
+---
+title: Type-safe configuration API
+summary: Inject configuration values based on annotations instead of using constants and Hadoop API
+date: 2019-04-25
+jira: HDDS-505
+status: implemented
+author: Anu Engineer, Marton Elek
+---
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Abstract
+
+ HA for Ozone Manager with the help of Ratis. High performance operation with caching and double-buffer.
+
+# Link
+
+ * https://issues.apache.org/jira/secure/attachment/12940314/OzoneManager%20HA.pdf
+
+ * https://issues.apache.org/jira/secure/attachment/12990063/OM%20HA%20Cache%20Design.pdf
+
+ * https://issues.apache.org/jira/secure/attachment/12973260/Handling%20Write%20Requests%20with%20OM%20HA.pdf
\ No newline at end of file
diff --git a/hadoop-hdds/docs/content/interface/OzoneFS.md b/hadoop-hdds/docs/content/interface/OzoneFS.md
index fcfef6d..c9fe4e2 100644
--- a/hadoop-hdds/docs/content/interface/OzoneFS.md
+++ b/hadoop-hdds/docs/content/interface/OzoneFS.md
@@ -145,11 +145,12 @@
The following table summarize which jar files and implementation should be used:
-Hadoop version | Required jar | OzoneFileSystem implementation
----------------|-------------------------|----------------------------------------------------
-3.2 | filesystem-lib-current | org.apache.hadoop.fs.ozone.OzoneFileSystem
-3.1 | filesystem-lib-legacy | org.apache.hadoop.fs.ozone.OzoneFileSystem
-2.9 | filesystem-lib-legacy | org.apache.hadoop.fs.ozone.BasicOzoneFileSystem
-2.7 | filesystem-lib-legacy | org.apache.hadoop.fs.ozone.BasicOzoneFileSystem
- With this method the hadoop-ozone-filesystem-lib-legacy.jar can be used from
+Hadoop version | Required jar | FileSystem implementation | AbstractFileSystem implementation
+---------------|-------------------------|-------------------------------------------------|-------------------------------------
+3.2 | filesystem-lib-current | org.apache.hadoop.fs.ozone.OzoneFileSystem | org.apache.hadoop.fs.ozone.OzFs
+3.1 | filesystem-lib-legacy | org.apache.hadoop.fs.ozone.OzoneFileSystem | org.apache.hadoop.fs.ozone.OzFs
+2.9 | filesystem-lib-legacy | org.apache.hadoop.fs.ozone.BasicOzoneFileSystem | org.apache.hadoop.fs.ozone.BasicOzFs
+2.7 | filesystem-lib-legacy | org.apache.hadoop.fs.ozone.BasicOzoneFileSystem | org.apache.hadoop.fs.ozone.BasicOzFs
+
+With this method the hadoop-ozone-filesystem-lib-legacy.jar can be used from
any older hadoop version (eg. hadoop 2.7 or spark+hadoop 2.7)
diff --git a/hadoop-hdds/docs/content/interface/OzoneFS.zh.md b/hadoop-hdds/docs/content/interface/OzoneFS.zh.md
index 17756e6..4e77f90 100644
--- a/hadoop-hdds/docs/content/interface/OzoneFS.zh.md
+++ b/hadoop-hdds/docs/content/interface/OzoneFS.zh.md
@@ -130,11 +130,11 @@
下表总结了各个版本 Hadoop 应当使用的 jar 包和文件系统实现：
-Hadoop 版本 | 需要的 jar | OzoneFileSystem 实现
----------------|-------------------------|----------------------------------------------------
-3.2 | filesystem-lib-current | org.apache.hadoop.fs.ozone.OzoneFileSystem
-3.1 | filesystem-lib-legacy | org.apache.hadoop.fs.ozone.OzoneFileSystem
-2.9 | filesystem-lib-legacy | org.apache.hadoop.fs.ozone.BasicOzoneFileSystem
-2.7 | filesystem-lib-legacy | org.apache.hadoop.fs.ozone.BasicOzoneFileSystem
+Hadoop 版本 | 需要的 jar | FileSystem 实现 | AbstractFileSystem 实现
+---------------|-------------------------|-------------------------------------------------|---------------------------
+3.2 | filesystem-lib-current | org.apache.hadoop.fs.ozone.OzoneFileSystem | org.apache.hadoop.fs.ozone.OzFs
+3.1 | filesystem-lib-legacy | org.apache.hadoop.fs.ozone.OzoneFileSystem | org.apache.hadoop.fs.ozone.OzFs
+2.9 | filesystem-lib-legacy | org.apache.hadoop.fs.ozone.BasicOzoneFileSystem | org.apache.hadoop.fs.ozone.BasicOzFs
+2.7 | filesystem-lib-legacy | org.apache.hadoop.fs.ozone.BasicOzoneFileSystem | org.apache.hadoop.fs.ozone.BasicOzFs
由此可知，低版本的 Hadoop 可以使用 hadoop-ozone-filesystem-lib-legacy.jar（比如 hadoop 2.7 或者 spark+hadoop 2.7）。
diff --git a/hadoop-hdds/docs/content/interface/S3.md b/hadoop-hdds/docs/content/interface/S3.md
index 6a8e2d7..94e4557 100644
--- a/hadoop-hdds/docs/content/interface/S3.md
+++ b/hadoop-hdds/docs/content/interface/S3.md
@@ -24,16 +24,24 @@
Ozone provides S3 compatible REST interface to use the object store data with any S3 compatible tools.
+S3 buckets are stored under the `/s3v`(Default is s3v, which can be setted through ozone.s3g.volume.name) volume, which needs to be created by an administrator first.
+
## Getting started
S3 Gateway is a separated component which provides the S3 compatible APIs. It should be started additional to the regular Ozone components.
You can start a docker based cluster, including the S3 gateway from the release package.
-Go to the `compose/ozones3` directory, and start the server:
+Go to the `compose/ozone` directory, and start the server:
```bash
-docker-compose up -d
+docker-compose up -d --scale datanode=3
+```
+
+Create the `/s3v` volume:
+
+```bash
+docker-compose exec scm ozone sh volume create /s3v
```
You can access the S3 gateway at `http://localhost:9878`
@@ -94,11 +102,10 @@
If security is enabled, you can get the key and the secret with the `ozone s3 getsecret` command (*kerberos based authentication is required).
```bash
-/etc/security/keytabs/testuser.keytab testuser/scm@EXAMPLE.COM
+kinit -kt /etc/security/keytabs/testuser.keytab testuser/scm@EXAMPLE.COM
ozone s3 getsecret
awsAccessKey=testuser/scm@EXAMPLE.COM
awsSecret=c261b6ecabf7d37d5f9ded654b1c724adac9bd9f13e247a235e567e8296d2999
-
```
Now, you can use the key and the secret to access the S3 endpoint:
@@ -109,21 +116,6 @@
aws s3api --endpoint http://localhost:9878 create-bucket --bucket bucket1
```
-
-## S3 bucket name mapping to Ozone buckets
-
-**Note**: Ozone has a notion for 'volumes' which is missing from the S3 Rest endpoint. Under the hood S3 bucket names are mapped to Ozone 'volume/bucket' locations (depending on the given authentication information).
-
-To show the storage location of a S3 bucket, use the `ozone s3 path <bucketname>` command.
-
-```bash
-aws s3api --endpoint-url http://localhost:9878 create-bucket --bucket=bucket1
-
-ozone s3 path bucket1
-Volume name for S3Bucket is : s3thisisakey
-Ozone FileSystem Uri is : o3fs://bucket1.s3thisisakey
-```
-
## Clients
### AWS Cli
diff --git a/hadoop-hdds/docs/content/interface/S3.zh.md b/hadoop-hdds/docs/content/interface/S3.zh.md
index 077124b..844c922 100644
--- a/hadoop-hdds/docs/content/interface/S3.zh.md
+++ b/hadoop-hdds/docs/content/interface/S3.zh.md
@@ -30,10 +30,10 @@
你可以使用发行包启动一个基于 docker 的集群，其中会包括 S3 网关。
-进入 `compose/ozones3` 目录，启动服务：
+进入 `compose/ozone` 目录，启动服务：
```bash
-docker-compose up -d
+docker-compose up -d --scale datanode=3
```
你可以通过 `http://localhost:9878` 访问 S3 网关。
@@ -94,11 +94,10 @@
在启用了安全机制的情况下，你可以通过 `ozone s3 gesecret` 命令获取 key 和 secret（需要进行 Kerberos 认证）。
```bash
-/etc/security/keytabs/testuser.keytab testuser/scm@EXAMPLE.COM
+kinit -kt /etc/security/keytabs/testuser.keytab testuser/scm@EXAMPLE.COM
ozone s3 getsecret
awsAccessKey=testuser/scm@EXAMPLE.COM
awsSecret=c261b6ecabf7d37d5f9ded654b1c724adac9bd9f13e247a235e567e8296d2999
-
```
现在你可以使用 key 和 secret 来访问 S3 endpoint：
@@ -110,20 +109,6 @@
```
-## S3 桶和 Ozone 桶的映射
-
-**注意**: Ozone 对 S3 REST 端点中缺失的卷都做了标记，在底层实现上，S3 的桶名被映射到 Ozone 桶的位置（取决于给定的认证信息）。
-
-要显示一个 S3 桶存储的位置，使用 `ozone s3 path <bucketname>` 命令。
-
-```bash
-aws s3api --endpoint-url http://localhost:9878 create-bucket --bucket=bucket1
-
-ozone s3 path bucket1
-Volume name for S3Bucket is : s3thisisakey
-Ozone FileSystem Uri is : o3fs://bucket1.s3thisisakey
-```
-
## 客户端
### AWS 命令行接口
diff --git a/hadoop-hdds/docs/content/recipe/Prometheus.zh.md b/hadoop-hdds/docs/content/recipe/Prometheus.zh.md
new file mode 100644
index 0000000..a235cd5
--- /dev/null
+++ b/hadoop-hdds/docs/content/recipe/Prometheus.zh.md
@@ -0,0 +1,93 @@
+---
+title: 使用 Prometheus 监控
+summary: 使用 Prometheus 监控 Ozone 的简要说明
+linktitle: Prometheus
+---
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+[Prometheus](https://prometheus.io/)是在[云原生计算基金会](https://www.cncf.io/)管理下开发的一款开源监控软件。
+
+Ozone 原生支持 Prometheus，各个组件会启动一个兼容 Prometheus 的指标端点，将所有可用的 hadoop 指标以 Prometheus 的格式导出发布。
+
+## 准备
+
+ 1. [安装启动]({{< ref "start/RunningViaDocker.zh.md" >}})一个 Ozone 集群。
+ 2. [下载](https://prometheus.io/download/#prometheus) Prometheus 二进制包。
+
+## 使用 Prometheus 进行监控
+
+* 你需要在 `ozone-site.xml` 文件中添加配置才可以启用 Prometheus 指标端点。
+
+ ```xml
+ <property>
+ <name>hdds.prometheus.endpoint.enabled</name>
+ <value>true</value>
+ </property>
+```
+
+_注意_: 对于基于 docker-compose 方式的伪集群，在 `docker-config` 文件中添加 `OZONE-SITE.XML_hdds.prometheus.endpoint.enabled=true`。
+
+* 重启 OM 和 SCM，检查端点：
+
+ * http://scm:9874/prom
+
+ * http://ozoneManager:9876/prom
+
+* 根据这两个端点，创建 prometheus.yaml 配置文件：
+
+```yaml
+global:
+ scrape_interval: 15s
+
+scrape_configs:
+ - job_name: ozone
+ metrics_path: /prom
+ static_configs:
+ - targets:
+ - "scm:9876"
+ - "ozoneManager:9874"
+```
+
+* 在 prometheus.yaml 文件所在目录启动 Prometheus：
+
+```bash
+prometheus
+```
+
+* 在 Prometheus 的 web ui 中查看目标：
+
+http://localhost:9090/targets
+
+
+
+
+* 在 Prometheus web ui 中查看任意的指标，例如：
+For example:
+
+http://localhost:9090/graph?g0.range_input=1h&g0.expr=om_metrics_num_key_allocate&g0.tab=1
+
+
+
+## 注意事项
+
+Ozone 发行包中包含了一个即开即用的容器化环境来试用 Ozone 和 Prometheus，在 `compose/ozoneperf` 目录下。
+
+```bash
+cd compose/ozoneperf
+docker-compose up -d
+```
diff --git a/hadoop-hdds/docs/content/recipe/SparkOzoneFSK8S.md b/hadoop-hdds/docs/content/recipe/SparkOzoneFSK8S.md
index 9f9d347..e8e9c60 100644
--- a/hadoop-hdds/docs/content/recipe/SparkOzoneFSK8S.md
+++ b/hadoop-hdds/docs/content/recipe/SparkOzoneFSK8S.md
@@ -112,26 +112,17 @@
docker push myrepo/spark-ozone
```
-## Create a bucket and identify the ozonefs path
+## Create a bucket
Download any text file and put it to the `/tmp/alice.txt` first.
```bash
kubectl port-forward s3g-0 9878:9878
+ozone sh volume create /s3v
aws s3api --endpoint http://localhost:9878 create-bucket --bucket=test
aws s3api --endpoint http://localhost:9878 put-object --bucket test --key alice.txt --body /tmp/alice.txt
-kubectl exec -it scm-0 ozone s3 path test
```
-The output of the last command is something like this:
-
-```
-Volume name for S3Bucket is : s3asdlkjqiskjdsks
-Ozone FileSystem Uri is : o3fs://test.s3asdlkjqiskjdsks
-```
-
-Write down the ozone filesystem uri as it should be used with the spark-submit command.
-
## Create service account to use
```bash
@@ -146,8 +137,6 @@
* the kubernetes namespace (_yournamespace_ in this example)
* serviceAccountName (you can use the _spark_ value if you followed the previous steps)
* container.image (in this example this is _myrepo/spark-ozone_. This is pushed to the registry in the previous steps)
- * location of the input file (o3fs://...), use the string which is identified earlier with the \
- `ozone s3 path <bucketname>` command
```bash
bin/spark-submit \
@@ -162,7 +151,7 @@
--conf spark.kubernetes.container.image.pullPolicy=Always \
--jars /opt/hadoop-ozone-filesystem-lib-legacy.jar \
local:///opt/spark/examples/jars/spark-examples_2.11-2.4.0.jar \
- o3fs://bucket.volume/alice.txt
+ o3fs://test.s3v/alice.txt
```
Check the available `spark-word-count-...` pods with `kubectl get pod`
diff --git a/hadoop-hdds/docs/content/recipe/_index.zh.md b/hadoop-hdds/docs/content/recipe/_index.zh.md
new file mode 100644
index 0000000..58e6012
--- /dev/null
+++ b/hadoop-hdds/docs/content/recipe/_index.zh.md
@@ -0,0 +1,28 @@
+---
+title: 使用配方
+date: "2017-10-10"
+menu: main
+weight: 9
+
+---
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+
+{{<jumbotron title="Ozone 使用配方">}}
+ 关于如何通过其它软件使用 Ozone 的标准文档，比如通过 Apache Spark 使用 Ozone。
+{{</jumbotron>}}
diff --git a/hadoop-hdds/docs/content/security/SecureOzone.md b/hadoop-hdds/docs/content/security/SecureOzone.md
index b33a603..7d3693e 100644
--- a/hadoop-hdds/docs/content/security/SecureOzone.md
+++ b/hadoop-hdds/docs/content/security/SecureOzone.md
@@ -164,11 +164,11 @@
</thead>
<tbody>
<tr>
- <td>ozone.s3g.authentication.kerberos.principal</th>
+ <td>ozone.s3g.http.auth.kerberos.principal</th>
<td>S3 Gateway principal. <br/> e.g. HTTP/_HOST@EXAMPLE.COM</td>
</tr>
<tr>
- <td>ozone.s3g.keytab.file</th>
+ <td>ozone.s3g.http.auth.kerberos.keytab</th>
<td>The keytab file used by S3 gateway</td>
</tr>
</tbody>
diff --git a/hadoop-hdds/docs/content/shell/VolumeCommands.md b/hadoop-hdds/docs/content/shell/VolumeCommands.md
index 47fb985..fe459f3 100644
--- a/hadoop-hdds/docs/content/shell/VolumeCommands.md
+++ b/hadoop-hdds/docs/content/shell/VolumeCommands.md
@@ -85,13 +85,15 @@
### List
-The `volume list` command will list the volumes owned by a user.
+The `volume list` command will list the volumes accessible by a user.
{{< highlight bash >}}
ozone sh volume list --user hadoop
{{< /highlight >}}
-The above command will print out all the volumes owned by the user hadoop.
+When ACL is enabled, the above command will print out volumes that the user
+hadoop has LIST permission to. When ACL is disabled, the above command will
+print out all the volumes owned by the user hadoop.
### Update
diff --git a/hadoop-hdds/docs/content/shell/VolumeCommands.zh.md b/hadoop-hdds/docs/content/shell/VolumeCommands.zh.md
index b4a4c28..190e099 100644
--- a/hadoop-hdds/docs/content/shell/VolumeCommands.zh.md
+++ b/hadoop-hdds/docs/content/shell/VolumeCommands.zh.md
@@ -80,13 +80,14 @@
### 列举
-`volume list` 命令用来列举一个用户拥有的所有卷。
+`volume list` 命令用来列举一个用户可以访问的所有卷。
{{< highlight bash >}}
ozone sh volume list --user hadoop
{{< /highlight >}}
-上述命令会打印出 hadoop 用户拥有的所有卷。
+若 ACL 已启用，上述命令会打印出 hadoop 用户有 LIST 权限的所有卷。
+若 ACL 被禁用，上述命令会打印出 hadoop 用户拥有的所有卷。
### 更新
diff --git a/hadoop-hdds/docs/content/start/StartFromDockerHub.md b/hadoop-hdds/docs/content/start/StartFromDockerHub.md
index e3e7d41..c4f36af 100644
--- a/hadoop-hdds/docs/content/start/StartFromDockerHub.md
+++ b/hadoop-hdds/docs/content/start/StartFromDockerHub.md
@@ -62,16 +62,21 @@
If you need multiple datanodes, we can just scale it up:
```bash
- docker-compose scale datanode=3
- ```
+docker-compose up -d --scale datanode=3
+```
+
# Running S3 Clients
Once the cluster is booted up and ready, you can verify its status by
connecting to the SCM's UI at [http://localhost:9876](http://localhost:9876).
The S3 gateway endpoint will be exposed at port 9878. You can use Ozone's S3
-support as if you are working against the real S3.
+support as if you are working against the real S3. S3 buckets are stored under
+the `/s3v` volume, which needs to be created by an administrator first:
+```
+docker-compose exec scm ozone sh volume create /s3v
+```
Here is how you create buckets from command line:
diff --git a/hadoop-hdds/docs/content/tools/AuditParser.zh.md b/hadoop-hdds/docs/content/tools/AuditParser.zh.md
new file mode 100644
index 0000000..10786d5
--- /dev/null
+++ b/hadoop-hdds/docs/content/tools/AuditParser.zh.md
@@ -0,0 +1,68 @@
+---
+title: "审计解析器"
+date: 2018-12-17
+summary: 审计解析器工具用来查看 Ozone 的审计日志。
+---
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+审计解析器工具用于查询 Ozone 的审计日志。它会在指定的路径下创建一个 sqlite 数据库，若数据库已存在则不再创建。
+
+这个数据库只包含了一个名为 `audit` 的表，定义如下：
+
+CREATE TABLE IF NOT EXISTS audit (
+datetime text,
+level varchar(7),
+logger varchar(7),
+user text,
+ip text,
+op text,
+params text,
+result varchar(7),
+exception text,
+UNIQUE(datetime,level,logger,user,ip,op,params,result))
+
+用法：
+{{< highlight bash >}}
+ozone auditparser <数据库文件的路径> [命令] [参数]
+{{< /highlight >}}
+
+将审计日志加载到数据库：
+{{< highlight bash >}}
+ozone auditparser <数据库文件的路径> load <审计日志的路径>
+{{< /highlight >}}
+Load 命令会创建如上所述的审计表。
+
+运行一个自定义的只读查询：
+{{< highlight bash >}}
+ozone auditparser <数据库文件的路径> query <双引号括起来的 select 查询>
+{{< /highlight >}}
+
+审计解析起自带了一些模板（最常用的查询）
+
+运行模板查询：
+{{< highlight bash >}}
+ozone auditparser <数据库文件的路径 template <模板名称>
+{{< /highlight >}}
+
+Ozone 提供了以下模板：
+
+|模板名称|描述|SQL|
+|----------------|----------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------|
+|top5users|Top 5 users|select user,count(*) as total from audit group by user order by total DESC limit 5|
+|top5cmds|Top 5 commands|select op,count(*) as total from audit group by op order by total DESC limit 5|
+|top5activetimebyseconds|Top 5 active times, grouped by seconds|select substr(datetime,1,charindex(',',datetime)-1) as dt,count(*) as thecount from audit group by dt order by thecount DESC limit 5|
diff --git a/hadoop-hdds/docs/content/tools/Genconf.zh.md b/hadoop-hdds/docs/content/tools/Genconf.zh.md
new file mode 100644
index 0000000..cd28269
--- /dev/null
+++ b/hadoop-hdds/docs/content/tools/Genconf.zh.md
@@ -0,0 +1,25 @@
+---
+title: 配置生成
+date: 2018-12-18
+summary: 用于生成默认配置的工具
+---
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+Genconf 工具可以在指定路径下生成 ozone-site.xml 模板文件，用户可以进行根据需求编辑此文件。
+
+`ozone genconf <path>`
diff --git a/hadoop-hdds/docs/content/tools/TestTools.md b/hadoop-hdds/docs/content/tools/TestTools.md
index a077f2e..63a014d 100644
--- a/hadoop-hdds/docs/content/tools/TestTools.md
+++ b/hadoop-hdds/docs/content/tools/TestTools.md
@@ -41,14 +41,14 @@
The smoketests are available from the distribution (`./smoketest`) but the robot files defines only the tests: usually they start CLI and check the output.
-To run the tests in different environmente (docker-compose, kubernetes) you need a definition to start the containers and execute the right tests in the right containers.
+To run the tests in different environment (docker-compose, kubernetes) you need a definition to start the containers and execute the right tests in the right containers.
These definition of the tests are included in the `compose` directory (check `./compose/*/test.sh` or `./compose/test-all.sh`).
-For example a simple way to test the distribution packege:
+For example a simple way to test the distribution package:
```
-cd compose/ozonze
+cd compose/ozone
./test.sh
```
@@ -56,7 +56,7 @@
[Blockade](https://github.com/worstcase/blockade) is a tool to test network failures and partitions (it's inspired by the legendary [Jepsen tests](https://jepsen.io/analyses)).
-Blockade tests are implemented with the help of tests and can be started from the `./blockade` directory of the distrubution.
+Blockade tests are implemented with the help of tests and can be started from the `./blockade` directory of the distribution.
```
cd blocakde
diff --git a/hadoop-hdds/docs/content/tools/TestTools.zh.md b/hadoop-hdds/docs/content/tools/TestTools.zh.md
new file mode 100644
index 0000000..81a6b08
--- /dev/null
+++ b/hadoop-hdds/docs/content/tools/TestTools.zh.md
@@ -0,0 +1,229 @@
+---
+title: "测试工具"
+summary: Ozone 提供了负载生成、网络分片测试、验收测试等多种测试工具。
+---
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+测试是开发分布式系统中最重要的部分，我们提供了以下类型的测试。
+
+本页面给出了 Ozone 自带的测试工具。
+
+注意：我们还进行了其它测试（比如通过 Spark 或 Hive 进行的 TCP-DS、TCP-H），但因为它们是外部工具，所以没有在此列出。
+
+## 单元测试
+
+和每个 java 项目一样，我们的每个项目都包含传统的单元测试。
+
+## 集成测试（JUnit）
+
+传统的单元测试只能测试一个单元，但我们也有更高层次的单元测试。它们使用 `MiniOzoneCluster` 辅助方法在单元测试中启动守护进程（SCM、OM、数据节点）。
+
+从 maven 或 java 的角度来看，集成测试也只是普通的单元测试而已（使用了 JUnit 库），但为了解决一些依赖问题，我们将它们单独放在了 `hadoop-ozone/integration-test` 目录下。
+
+## 冒烟测试
+
+我们使用基于 docker-compose 的伪集群来运行不同配置的 Ozone，为了确保这些配置可用，我们在 https://robotframework.org/ 的帮助下实现了 _验收_ 测试。
+
+冒烟测试包含在发行包中（`./smoketest`），但 robot 文件只定义了运行命令行然后检查输出的测试。
+
+为了在不同环境（docker-compose、kubernetes）下运行冒烟测试，你需要定义如何启动容器，然后在正确的容器中执行正确的测试。
+
+这部分的测试包含在 `compose` 目录中（查看 `./compose/*/test.sh` 或者 `./compose/test-all.sh`）。
+
+例如，一种测试分发包的简单方法是：
+
+```
+cd compose/ozone
+./test.sh
+```
+
+## Blockade
+
+[Blockade](https://github.com/worstcase/blockade) 是一个测试网络故障和分片的工具（灵感来自于大名鼎鼎的[Jepsen 测试](https://jepsen.io/analyses)）。
+
+Blockade 测试在其它测试的基础上实现，可以在分发包中的 `./blockade` 目录下进行测试。
+
+```
+cd blocakde
+pip install pytest==2.8.7,blockade
+python -m pytest -s .
+```
+
+更多细节查看 blockade 目录下的 README。
+
+## MiniChaosOzoneCluster
+
+这是一种在你的机器上获得[混沌](https://en.wikipedia.org/wiki/Chaos_engineering)的方法。它可以直接从源码启动一个 MiniOzoneCluster
+（会启动真实的守护进程），并随机杀死它。
+
+## Freon
+
+Freon 是 Ozone 发行包中包含的命令行应用，它是一个负载生成器，用于压力测试。
+
+例如：
+
+```
+ozone freon randomkeys --numOfVolumes=10 --numOfBuckets 10 --numOfKeys 10 --replicationType=RATIS --factor=THREE
+```
+
+```
+***************************************************
+Status: Success
+Git Base Revision: 48aae081e5afacbb3240657556b26c29e61830c3
+Number of Volumes created: 10
+Number of Buckets created: 100
+Number of Keys added: 1000
+Ratis replication factor: THREE
+Ratis replication type: RATIS
+Average Time spent in volume creation: 00:00:00,035
+Average Time spent in bucket creation: 00:00:00,319
+Average Time spent in key creation: 00:00:03,659
+Average Time spent in key write: 00:00:10,894
+Total bytes written: 10240000
+Total Execution time: 00:00:16,898
+***********************
+```
+
+更多细节请查看 [freon 文档页面](https://hadoop.apache.org/ozone/docs/0.4.0-alpha/freon.html)
+
+## Genesis
+
+Genesis 是一个微型的基准测试工具，它也包含在发行包中（`ozone genesis`），但是它不需要一个真实的集群，而是采用一种隔离的方法测试不同部分的代码（比如，将数据存储到本地基于 RocksDB 的键值存储中）。
+
+运行示例：
+
+```
+ ozone genesis -benchmark=BenchMarkRocksDbStore
+# JMH version: 1.19
+# VM version: JDK 11.0.1, VM 11.0.1+13-LTS
+# VM invoker: /usr/lib/jvm/java-11-openjdk-11.0.1.13-3.el7_6.x86_64/bin/java
+# VM options: -Dproc_genesis -Djava.net.preferIPv4Stack=true -Dhadoop.log.dir=/var/log/hadoop -Dhadoop.log.file=hadoop.log -Dhadoop.home.dir=/opt/hadoop -Dhadoop.id.str=hadoop -Dhadoop.root.logger=INFO,console -Dhadoop.policy.file=hadoop-policy.xml -Dhadoop.security.logger=INFO,NullAppender
+# Warmup: 2 iterations, 1 s each
+# Measurement: 20 iterations, 1 s each
+# Timeout: 10 min per iteration
+# Threads: 4 threads, will synchronize iterations
+# Benchmark mode: Throughput, ops/time
+# Benchmark: org.apache.hadoop.ozone.genesis.BenchMarkRocksDbStore.test
+# Parameters: (backgroundThreads = 4, blockSize = 8, maxBackgroundFlushes = 4, maxBytesForLevelBase = 512, maxOpenFiles = 5000, maxWriteBufferNumber = 16, writeBufferSize = 64)
+
+# Run progress: 0.00% complete, ETA 00:00:22
+# Fork: 1 of 1
+# Warmup Iteration 1: 213775.360 ops/s
+# Warmup Iteration 2: 32041.633 ops/s
+Iteration 1: 196342.348 ops/s
+ ?stack: <delayed till summary>
+
+Iteration 2: 41926.816 ops/s
+ ?stack: <delayed till summary>
+
+Iteration 3: 210433.231 ops/s
+ ?stack: <delayed till summary>
+
+Iteration 4: 46941.951 ops/s
+ ?stack: <delayed till summary>
+
+Iteration 5: 212825.884 ops/s
+ ?stack: <delayed till summary>
+
+Iteration 6: 145914.351 ops/s
+ ?stack: <delayed till summary>
+
+Iteration 7: 141838.469 ops/s
+ ?stack: <delayed till summary>
+
+Iteration 8: 205334.438 ops/s
+ ?stack: <delayed till summary>
+
+Iteration 9: 163709.519 ops/s
+ ?stack: <delayed till summary>
+
+Iteration 10: 162494.608 ops/s
+ ?stack: <delayed till summary>
+
+Iteration 11: 199155.793 ops/s
+ ?stack: <delayed till summary>
+
+Iteration 12: 209679.298 ops/s
+ ?stack: <delayed till summary>
+
+Iteration 13: 193787.574 ops/s
+ ?stack: <delayed till summary>
+
+Iteration 14: 127004.147 ops/s
+ ?stack: <delayed till summary>
+
+Iteration 15: 145511.080 ops/s
+ ?stack: <delayed till summary>
+
+Iteration 16: 223433.864 ops/s
+ ?stack: <delayed till summary>
+
+Iteration 17: 169752.665 ops/s
+ ?stack: <delayed till summary>
+
+Iteration 18: 165217.191 ops/s
+ ?stack: <delayed till summary>
+
+Iteration 19: 191038.476 ops/s
+ ?stack: <delayed till summary>
+
+Iteration 20: 196335.579 ops/s
+ ?stack: <delayed till summary>
+
+
+
+Result "org.apache.hadoop.ozone.genesis.BenchMarkRocksDbStore.test":
+ 167433.864 ?(99.9%) 43530.883 ops/s [Average]
+ (min, avg, max) = (41926.816, 167433.864, 223433.864), stdev = 50130.230
+ CI (99.9%): [123902.981, 210964.748] (assumes normal distribution)
+
+Secondary result "org.apache.hadoop.ozone.genesis.BenchMarkRocksDbStore.test:?stack":
+Stack profiler:
+
+....[Thread state distributions]....................................................................
+ 78.9% RUNNABLE
+ 20.0% TIMED_WAITING
+ 1.1% WAITING
+
+....[Thread state: RUNNABLE]........................................................................
+ 59.8% 75.8% org.rocksdb.RocksDB.put
+ 16.5% 20.9% org.rocksdb.RocksDB.get
+ 0.7% 0.9% java.io.UnixFileSystem.delete0
+ 0.7% 0.9% org.rocksdb.RocksDB.disposeInternal
+ 0.3% 0.4% java.lang.Long.formatUnsignedLong0
+ 0.1% 0.2% org.apache.hadoop.ozone.genesis.BenchMarkRocksDbStore.test
+ 0.1% 0.1% java.lang.Long.toUnsignedString0
+ 0.1% 0.1% org.apache.hadoop.ozone.genesis.generated.BenchMarkRocksDbStore_test_jmhTest.test_thrpt_jmhStub
+ 0.0% 0.1% java.lang.Object.clone
+ 0.0% 0.0% java.lang.Thread.currentThread
+ 0.4% 0.5% <other>
+
+....[Thread state: TIMED_WAITING]...................................................................
+ 20.0% 100.0% java.lang.Object.wait
+
+....[Thread state: WAITING].........................................................................
+ 1.1% 100.0% jdk.internal.misc.Unsafe.park
+
+
+
+# Run complete. Total time: 00:00:38
+
+Benchmark (backgroundThreads) (blockSize) (maxBackgroundFlushes) (maxBytesForLevelBase) (maxOpenFiles) (maxWriteBufferNumber) (writeBufferSize) Mode Cnt Score Error Units
+BenchMarkRocksDbStore.test 4 8 4 512 5000 16 64 thrpt 20 167433.864 ? 43530.883 ops/s
+BenchMarkRocksDbStore.test:?stack 4 8 4 512 5000 16 64 thrpt NaN ---
+```
diff --git a/hadoop-hdds/docs/pom.xml b/hadoop-hdds/docs/pom.xml
index cb600f2..4c16606 100644
--- a/hadoop-hdds/docs/pom.xml
+++ b/hadoop-hdds/docs/pom.xml
@@ -36,7 +36,6 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
- <version>1.6.0</version>
<executions>
<execution>
<goals>
diff --git a/hadoop-hdds/docs/themes/ozonedoc/layouts/_default/baseof.html b/hadoop-hdds/docs/themes/ozonedoc/layouts/_default/baseof.html
new file mode 100644
index 0000000..c46f829
--- /dev/null
+++ b/hadoop-hdds/docs/themes/ozonedoc/layouts/_default/baseof.html
@@ -0,0 +1,41 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+{{ partial "header.html" . }}
+
+<body>
+
+{{ partial "navbar.html" . }}
+
+<div class="container-fluid">
+ <div class="row">
+ {{ partial "sidebar.html" . }}
+ <div class="col-sm-9 col-sm-offset-3 col-md-10 col-md-offset-2 main">
+
+ <main>
+ {{ block "main" .}}
+ {{end}}
+ </main>
+
+ </div>
+ </div>
+</div>
+</div>
+
+{{ partial "footer.html" . }}
+
+</body>
+</html>
diff --git a/hadoop-hdds/docs/themes/ozonedoc/layouts/design/section.html b/hadoop-hdds/docs/themes/ozonedoc/layouts/design/section.html
new file mode 100644
index 0000000..a701518
--- /dev/null
+++ b/hadoop-hdds/docs/themes/ozonedoc/layouts/design/section.html
@@ -0,0 +1,68 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+{{ partial "header.html" . }}
+
+<body>
+
+{{ partial "navbar.html" . }}
+
+<div class="container-fluid">
+ <div class="row">
+ {{ partial "sidebar.html" . }}
+ <div class="col-sm-9 col-sm-offset-3 col-md-10 col-md-offset-2 main">
+
+ <main>
+ <main>
+ <h1>Available design docs</h1>
+ {{ .Content }}
+ <table class="table">
+ <thead>
+ <tr>
+ <td>Date</td>
+ <td>JIRA</td>
+ <td>Title</td>
+ <td>Summary</td>
+ <td>Status</td>
+ </tr>
+ </thead>
+ <tbody>
+ {{ range .Pages }}
+ <tr>
+ <td>{{.Params.date | dateFormat "2006-01-02"}}</td>
+ <td>
+ <a href="https://issues.apache.org/jira/browse/{{.Params.jira}}">{{.Params.jira}}</a>
+ </td>
+ <td><a href="{{.URL}}">{{.Title}}</a></td>
+ <td>{{.Summary}}</td>
+ <td>{{.Params.status}}</td>
+ </tr>
+ {{ end }}
+ </tbody>
+ </table>
+ </main>
+ </main>
+
+ </div>
+ </div>
+</div>
+</div>
+
+{{ partial "footer.html" . }}
+
+</body>
+</html>
+
diff --git a/hadoop-hdds/docs/themes/ozonedoc/layouts/design/single.html b/hadoop-hdds/docs/themes/ozonedoc/layouts/design/single.html
new file mode 100644
index 0000000..f92f226
--- /dev/null
+++ b/hadoop-hdds/docs/themes/ozonedoc/layouts/design/single.html
@@ -0,0 +1,47 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+{{ partial "header.html" . }}
+
+ <body>
+{{ partial "navbar.html" . }}
+ <div class="container-fluid">
+ <div class="row">
+ {{ partial "sidebar.html" . }}
+ <div class="col-sm-9 col-sm-offset-3 col-md-10 col-md-offset-2 main">
+ <div class="col-md-9">
+ <h1><a href="https://issues.apache.org/jira/browse/{{.Params.jira}}">[{{.Params.jira}}]</a> {{ .Title }} ({{.Params.status}}) </h1>
+ <div><i>Authors: {{.Params.author}}</i><div class="pull-right">{{.Params.date | dateFormat "2006-01-02"}}</div></div>
+ <p> </p>
+
+ <div class="panel panel-success">
+ <div class="panel-heading">Summary</div>
+ <div class="panel-body">
+ {{.Summary}}
+ </div>
+ </div>
+
+ {{ .Content }}
+ </div>
+
+ </div>
+ </div>
+ </div>
+
+{{ partial "footer.html" . }}
+
+ </body>
+</html>
diff --git a/hadoop-hdds/docs/themes/ozonedoc/layouts/partials/sidebar.html b/hadoop-hdds/docs/themes/ozonedoc/layouts/partials/sidebar.html
index e065f15..f4faf2f 100644
--- a/hadoop-hdds/docs/themes/ozonedoc/layouts/partials/sidebar.html
+++ b/hadoop-hdds/docs/themes/ozonedoc/layouts/partials/sidebar.html
@@ -60,6 +60,7 @@
</li>
{{ end }}
{{ end }}
+ <li><a href="{{ "design.html" | relURL }}"><span><b>Design docs</b></span></a></li>
<li class="visible-xs"><a href="#">References</a>
<ul class="nav">
<li><a href="https://github.com/apache/hadoop"><span class="glyphicon glyphicon-new-window" aria-hidden="true"></span> Source</a></li>
diff --git a/hadoop-hdds/framework/pom.xml b/hadoop-hdds/framework/pom.xml
index 7fdf3e9..0333d67 100644
--- a/hadoop-hdds/framework/pom.xml
+++ b/hadoop-hdds/framework/pom.xml
@@ -32,17 +32,54 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-test-utils</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-common</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-hadoop-dependency-server</artifactId>
+ </dependency>
+ <dependency>
+ <artifactId>ratis-server</artifactId>
+ <groupId>org.apache.ratis</groupId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcprov-jdk15on</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.rocksdb</groupId>
+ <artifactId>rocksdbjni</artifactId>
+ </dependency>
+ <dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_dropwizard</artifactId>
- <version>0.7.0</version>
</dependency>
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_common</artifactId>
- <version>0.7.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.datatype</groupId>
+ <artifactId>jackson-datatype-jsr310</artifactId>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
@@ -51,15 +88,12 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
+ <artifactId>hadoop-hdds-hadoop-dependency-test</artifactId>
<scope>test</scope>
- <type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-client</artifactId>
- <version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>com.squareup.okhttp</groupId>
@@ -70,7 +104,6 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop.version}</version>
<scope>test</scope>
<type>test-jar</type>
</dependency>
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisGrpcConfig.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisGrpcConfig.java
similarity index 100%
rename from hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisGrpcConfig.java
rename to hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisGrpcConfig.java
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisServerConfig.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisServerConfig.java
similarity index 100%
rename from hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisServerConfig.java
rename to hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/DatanodeRatisServerConfig.java
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/HddsConfServlet.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/HddsConfServlet.java
similarity index 93%
rename from hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/HddsConfServlet.java
rename to hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/HddsConfServlet.java
index 2c5c817..368197a 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/HddsConfServlet.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/HddsConfServlet.java
@@ -17,30 +17,27 @@
*/
package org.apache.hadoop.hdds.conf;
-import com.google.gson.Gson;
-import java.io.IOException;
-import java.io.Writer;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.core.HttpHeaders;
+import java.io.IOException;
+import java.io.Writer;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.annotation.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpServer2;
import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.Gson;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_TAGS_SYSTEM_KEY;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_TAGS_SYSTEM_KEY;
-
/**
* A servlet to print out the running configuration data.
*/
@@ -63,8 +60,9 @@
* Return the Configuration of the daemon hosting this servlet.
* This is populated when the HttpServer starts.
*/
- private Configuration getConfFromContext() {
- Configuration conf = (Configuration) getServletContext().getAttribute(
+ private OzoneConfiguration getConfFromContext() {
+ OzoneConfiguration conf =
+ (OzoneConfiguration) getServletContext().getAttribute(
HttpServer2.CONF_CONTEXT_ATTRIBUTE);
assert conf != null;
return conf;
@@ -127,11 +125,11 @@
/**
* Guts of the servlet - extracted for easy testing.
*/
- static void writeResponse(Configuration conf,
+ static void writeResponse(OzoneConfiguration conf,
Writer out, String format, String propertyName)
throws IOException, IllegalArgumentException, BadFormatException {
if (FORMAT_JSON.equals(format)) {
- Configuration.dumpConfiguration(conf, propertyName, out);
+ OzoneConfiguration.dumpConfiguration(conf, propertyName, out);
} else if (FORMAT_XML.equals(format)) {
conf.writeXml(propertyName, out);
} else {
@@ -155,7 +153,7 @@
Writer out) throws IOException {
String cmd = request.getParameter(COMMAND);
Gson gson = new Gson();
- Configuration config = getOzoneConfig();
+ OzoneConfiguration config = getOzoneConfig();
switch (cmd) {
case "getOzoneTags":
@@ -184,7 +182,7 @@
}
- private static Configuration getOzoneConfig() {
+ private static OzoneConfiguration getOzoneConfig() {
return OZONE_CONFIG;
}
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/package-info.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/package-info.java
similarity index 90%
copy from hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/package-info.java
copy to hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/package-info.java
index 80c1985..948057e 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/package-info.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/package-info.java
@@ -15,7 +15,4 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.ozone.web.ozShell;
-/**
- * Tests for ozone shell..
- */
+package org.apache.hadoop.hdds.conf;
\ No newline at end of file
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/CertificateSignRequest.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/CertificateSignRequest.java
index 21a19b5..f740e43 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/CertificateSignRequest.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/CertificateSignRequest.java
@@ -18,12 +18,21 @@
*/
package org.apache.hadoop.hdds.security.x509.certificates.utils;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
+import java.io.IOException;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.security.KeyPair;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil;
+
+import com.google.common.base.Preconditions;
import org.apache.logging.log4j.util.Strings;
import org.bouncycastle.asn1.ASN1EncodableVector;
import org.bouncycastle.asn1.ASN1Object;
@@ -50,14 +59,6 @@
import org.bouncycastle.util.io.pem.PemObject;
import org.bouncycastle.util.io.pem.PemReader;
-import java.io.IOException;
-import java.io.StringReader;
-import java.io.StringWriter;
-import java.security.KeyPair;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-
/**
* A certificate sign request object that wraps operations to build a
* PKCS10CertificationRequest to CertificateServer.
@@ -154,7 +155,7 @@
private boolean digitalEncryption;
public CertificateSignRequest.Builder setConfiguration(
- Configuration configuration) {
+ ConfigurationSource configuration) {
this.config = new SecurityConfig(configuration);
return this;
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/SelfSignedCertificate.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/SelfSignedCertificate.java
index 1fd6d7c..7ecc161 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/SelfSignedCertificate.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/SelfSignedCertificate.java
@@ -19,13 +19,23 @@
package org.apache.hadoop.hdds.security.x509.certificates.utils;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.security.KeyPair;
+import java.time.Duration;
+import java.time.LocalDate;
+import java.time.LocalTime;
+import java.time.ZoneOffset;
+import java.util.Date;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
import org.apache.hadoop.util.Time;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import org.apache.logging.log4j.util.Strings;
import org.bouncycastle.asn1.DEROctetString;
import org.bouncycastle.asn1.x500.X500Name;
@@ -40,15 +50,6 @@
import org.bouncycastle.operator.OperatorCreationException;
import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
-import java.io.IOException;
-import java.math.BigInteger;
-import java.security.KeyPair;
-import java.time.Duration;
-import java.time.LocalDate;
-import java.time.LocalTime;
-import java.time.ZoneOffset;
-import java.util.Date;
-
/**
* A Self Signed Certificate with CertificateServer basic constraint can be used
* to bootstrap a certificate infrastructure, if no external certificate is
@@ -158,7 +159,7 @@
private SecurityConfig config;
private boolean isCA;
- public Builder setConfiguration(Configuration configuration) {
+ public Builder setConfiguration(ConfigurationSource configuration) {
this.config = new SecurityConfig(configuration);
return this;
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
index 640f5ca..1f3b665 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
@@ -18,16 +18,17 @@
*/
package org.apache.hadoop.hdds.security.x509.keys;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.security.KeyPair;
import java.security.KeyPairGenerator;
import java.security.NoSuchAlgorithmException;
import java.security.NoSuchProviderException;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* A class to generate Key Pair for use with Certificates.
*/
@@ -41,7 +42,7 @@
*
* @param configuration - config
*/
- public HDDSKeyGenerator(Configuration configuration) {
+ public HDDSKeyGenerator(ConfigurationSource configuration) {
this.securityConfig = new SecurityConfig(configuration);
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/web/utils/JsonUtils.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/JsonUtils.java
similarity index 97%
rename from hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/web/utils/JsonUtils.java
rename to hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/JsonUtils.java
index b9abcf9..79c3b4e 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/web/utils/JsonUtils.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/JsonUtils.java
@@ -16,7 +16,10 @@
* limitations under the License.
*/
-package org.apache.hadoop.ozone.web.utils;
+package org.apache.hadoop.hdds.server;
+
+import java.io.IOException;
+import java.util.List;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
@@ -24,9 +27,6 @@
import com.fasterxml.jackson.databind.type.CollectionType;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
-import java.io.IOException;
-import java.util.List;
-
/**
* JSON Utility functions used in ozone.
*/
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/OzoneProtocolMessageDispatcher.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/OzoneProtocolMessageDispatcher.java
index fb13fe2..c239c0d 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/OzoneProtocolMessageDispatcher.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/OzoneProtocolMessageDispatcher.java
@@ -23,7 +23,7 @@
import com.google.protobuf.ProtocolMessageEnum;
import com.google.protobuf.ServiceException;
-import io.opentracing.Scope;
+import io.opentracing.Span;
import org.slf4j.Logger;
/**
@@ -38,12 +38,14 @@
private String serviceName;
- private final ProtocolMessageMetrics protocolMessageMetrics;
+ private final ProtocolMessageMetrics<ProtocolMessageEnum>
+ protocolMessageMetrics;
private Logger logger;
public OzoneProtocolMessageDispatcher(String serviceName,
- ProtocolMessageMetrics protocolMessageMetrics, Logger logger) {
+ ProtocolMessageMetrics<ProtocolMessageEnum> protocolMessageMetrics,
+ Logger logger) {
this.serviceName = serviceName;
this.protocolMessageMetrics = protocolMessageMetrics;
this.logger = logger;
@@ -54,8 +56,7 @@
FunctionWithServiceException<REQUEST, RESPONSE> methodCall,
ProtocolMessageEnum type,
String traceId) throws ServiceException {
- Scope scope = TracingUtil
- .importAndCreateScope(type.toString(), traceId);
+ Span span = TracingUtil.importAndCreateSpan(type.toString(), traceId);
try {
if (logger.isTraceEnabled()) {
logger.trace(
@@ -67,10 +68,13 @@
logger.debug("{} {} request is received",
serviceName, type.toString());
}
- protocolMessageMetrics.increment(type);
+
+ long startTime = System.nanoTime();
RESPONSE response = methodCall.apply(request);
+ protocolMessageMetrics.increment(type, System.nanoTime() - startTime);
+
if (logger.isTraceEnabled()) {
logger.trace(
"[service={}] [type={}] request is processed. Response: "
@@ -82,7 +86,7 @@
return response;
} finally {
- scope.close();
+ span.finish();
}
}
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java
index 9c87018..08f04f1 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java
@@ -21,8 +21,8 @@
import java.net.InetSocketAddress;
import java.util.Collection;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.ipc.RPC;
@@ -138,7 +138,7 @@
* @param conf
* @return
*/
- public static File getScmDbDir(Configuration conf) {
+ public static File getScmDbDir(ConfigurationSource conf) {
File metadataDir = getDirectoryFromConfig(conf,
ScmConfigKeys.OZONE_SCM_DB_DIRS, "SCM");
if (metadataDir != null) {
@@ -159,7 +159,7 @@
* @param componentName Which component's key is this
* @return File created from the value of the key in conf.
*/
- public static File getDirectoryFromConfig(Configuration conf,
+ public static File getDirectoryFromConfig(ConfigurationSource conf,
String key,
String componentName) {
final Collection<String> metadirs = conf.getTrimmedStringCollection(key);
@@ -191,7 +191,7 @@
* @return File MetaDir
* @throws IllegalArgumentException if the configuration setting is not set
*/
- public static File getOzoneMetaDirPath(Configuration conf) {
+ public static File getOzoneMetaDirPath(ConfigurationSource conf) {
File dirPath = getDirectoryFromConfig(conf,
HddsConfigKeys.OZONE_METADATA_DIRS, "Ozone");
if (dirPath == null) {
@@ -215,7 +215,7 @@
* @param key The configuration key which specify the directory.
* @return The path of the directory.
*/
- public static File getDBPath(Configuration conf, String key) {
+ public static File getDBPath(ConfigurationSource conf, String key) {
final File dbDirPath =
getDirectoryFromConfig(conf, key, "OM");
if (dbDirPath != null) {
@@ -233,7 +233,7 @@
return remoteUser != null ? remoteUser.getUserName() : null;
}
- public static String getDefaultRatisDirectory(Configuration conf) {
+ public static String getDefaultRatisDirectory(ConfigurationSource conf) {
LOG.warn("Storage directory for Ratis is not configured. It is a good " +
"idea to map this to an SSD disk. Falling back to {}",
HddsConfigKeys.OZONE_METADATA_DIRS);
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/EventQueue.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/EventQueue.java
index 6991554..3d7f06b 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/EventQueue.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/EventQueue.java
@@ -17,17 +17,6 @@
*/
package org.apache.hadoop.hdds.server.events;
-import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.Time;
-
-import com.google.common.base.Preconditions;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -37,6 +26,16 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Time;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* Simple async event processing utility.
* <p>
@@ -204,7 +203,7 @@
* all the dependent messages (messages which are sent by current handlers)
* are processed.
*
- * @param timeout Timeout in seconds to wait for the processing.
+ * @param timeout Timeout in milliseconds to wait for the processing.
*/
@VisibleForTesting
public void processAll(long timeout) {
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/BaseHttpServer.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/BaseHttpServer.java
index 86de95e..9cb4992 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/BaseHttpServer.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/BaseHttpServer.java
@@ -27,11 +27,15 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.DFSConfigKeysLegacy;
import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.HddsConfServlet;
+import org.apache.hadoop.hdds.conf.HddsPrometheusConfig;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.ozone.OzoneSecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
@@ -42,9 +46,14 @@
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_HTTPS_NEED_AUTH_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_HTTPS_NEED_AUTH_KEY;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_HTTP_SECURITY_ENABLED_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_HTTP_SECURITY_ENABLED_KEY;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SERVER_HTTPS_KEYPASSWORD_KEY;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY;
+
import org.eclipse.jetty.webapp.WebAppContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -61,7 +70,7 @@
"org.eclipse.jetty.webapp.basetempdir";
private HttpServer2 httpServer;
- private final Configuration conf;
+ private final ConfigurationSource conf;
private InetSocketAddress httpAddress;
private InetSocketAddress httpsAddress;
@@ -75,7 +84,8 @@
private boolean profilerSupport;
- public BaseHttpServer(Configuration conf, String name) throws IOException {
+ public BaseHttpServer(ConfigurationSource conf, String name)
+ throws IOException {
this.name = name;
this.conf = conf;
policy = getHttpPolicy(conf);
@@ -86,11 +96,32 @@
// Avoid registering o.a.h.http.PrometheusServlet in HttpServer2.
// TODO: Replace "hadoop.prometheus.endpoint.enabled" with
// CommonConfigurationKeysPublic.HADOOP_PROMETHEUS_ENABLED when possible.
- conf.setBoolean("hadoop.prometheus.endpoint.enabled", false);
+ conf.set("hadoop.prometheus.endpoint.enabled", "false");
HttpServer2.Builder builder = newHttpServer2BuilderForOzone(
- conf, httpAddress, httpsAddress,
- name, getSpnegoPrincipal(), getKeytabFile());
+ conf, httpAddress, httpsAddress, name);
+
+ boolean isSecurityEnabled = UserGroupInformation.isSecurityEnabled() &&
+ OzoneSecurityUtil.isHttpSecurityEnabled(conf);
+ LOG.info("Hadoop Security Enabled: {} " +
+ "Ozone Security Enabled: {} " +
+ "Ozone HTTP Security Enabled: {} ",
+ UserGroupInformation.isSecurityEnabled(),
+ conf.getBoolean(OZONE_SECURITY_ENABLED_KEY,
+ OZONE_SECURITY_ENABLED_DEFAULT),
+ conf.getBoolean(OZONE_HTTP_SECURITY_ENABLED_KEY,
+ OZONE_HTTP_SECURITY_ENABLED_DEFAULT));
+
+ if (isSecurityEnabled) {
+ String httpAuthType = conf.get(getHttpAuthType(), "simple");
+ LOG.info("HttpAuthType: {} = {}", getHttpAuthType(), httpAuthType);
+ if (httpAuthType.equals("kerberos")) {
+ builder.setSecurityEnabled(true);
+ builder.authFilterConfigurationPrefix(getHttpAuthConfigPrefix());
+ builder.setUsernameConfKey(getSpnegoPrincipal());
+ builder.setKeytabConfKey(getKeytabFile());
+ }
+ }
final boolean xFrameEnabled = conf.getBoolean(
DFSConfigKeysLegacy.DFS_XFRAME_OPTION_ENABLED,
@@ -116,7 +147,23 @@
prometheusMetricsSink = new PrometheusMetricsSink();
httpServer.getWebAppContext().getServletContext()
.setAttribute(PROMETHEUS_SINK, prometheusMetricsSink);
- httpServer.addServlet("prometheus", "/prom", PrometheusServlet.class);
+ HddsPrometheusConfig prometheusConfig =
+ OzoneConfiguration.of(conf).getObject(HddsPrometheusConfig.class);
+ String token = prometheusConfig.getPrometheusEndpointToken();
+ if (StringUtils.isNotEmpty(token)) {
+ httpServer.getWebAppContext().getServletContext()
+ .setAttribute(PrometheusServlet.SECURITY_TOKEN, token);
+ // Adding as internal servlet since we want to have token based
+ // auth and hence SPNEGO should be disabled if security is enabled.
+ httpServer.addInternalServlet("prometheus", "/prom",
+ PrometheusServlet.class);
+ } else {
+ // If token is not configured, keeping as regular servlet and not
+ // internal servlet since we do not want to expose /prom endpoint
+ // without authentication in a secure cluster.
+ httpServer.addServlet("prometheus", "/prom",
+ PrometheusServlet.class);
+ }
}
if (profilerSupport) {
@@ -134,32 +181,20 @@
}
}
-
-
/**
* Return a HttpServer.Builder that the OzoneManager/SCM/Datanode/S3Gateway/
* Recon to initialize their HTTP / HTTPS server.
*/
public static HttpServer2.Builder newHttpServer2BuilderForOzone(
- Configuration conf, final InetSocketAddress httpAddr,
- final InetSocketAddress httpsAddr, String name, String spnegoUserNameKey,
- String spnegoKeytabFileKey) throws IOException {
+ ConfigurationSource conf, final InetSocketAddress httpAddr,
+ final InetSocketAddress httpsAddr, String name) throws IOException {
HttpConfig.Policy policy = getHttpPolicy(conf);
HttpServer2.Builder builder = new HttpServer2.Builder().setName(name)
.setConf(conf).setACL(new AccessControlList(conf.get(
- OZONE_ADMINISTRATORS, " ")))
- .setSecurityEnabled(UserGroupInformation.isSecurityEnabled())
- .setUsernameConfKey(spnegoUserNameKey)
- .setKeytabConfKey(spnegoKeytabFileKey);
+ OZONE_ADMINISTRATORS, " ")));
// initialize the webserver for uploading/downloading files.
- if (UserGroupInformation.isSecurityEnabled()) {
- LOG.info("Starting web server as: "
- + SecurityUtil.getServerPrincipal(conf.get(spnegoUserNameKey),
- httpAddr.getHostName()));
- }
-
if (policy.isHttpEnabled()) {
if (httpAddr.getPort() == 0) {
builder.setFindPort(true);
@@ -171,7 +206,7 @@
}
if (policy.isHttpsEnabled() && httpsAddr != null) {
- Configuration sslConf = loadSslConfiguration(conf);
+ ConfigurationSource sslConf = loadSslConfiguration(conf);
loadSslConfToHttpServerBuilder(builder, sslConf);
if (httpsAddr.getPort() == 0) {
@@ -198,6 +233,12 @@
httpServer.addServlet(servletName, pathSpec, clazz);
}
+ protected void addInternalServlet(String servletName, String pathSpec,
+ Class<? extends HttpServlet> clazz) {
+ httpServer.addInternalServlet(servletName, pathSpec, clazz);
+ }
+
+
/**
* Returns the WebAppContext associated with this HttpServer.
*
@@ -294,7 +335,7 @@
public static HttpServer2.Builder loadSslConfToHttpServerBuilder(
- HttpServer2.Builder builder, Configuration sslConf) {
+ HttpServer2.Builder builder, ConfigurationSource sslConf) {
return builder
.needsClientAuth(
sslConf.getBoolean(OZONE_CLIENT_HTTPS_NEED_AUTH_KEY,
@@ -310,25 +351,6 @@
sslConf.get("ssl.server.exclude.cipher.list"));
}
-
- /**
- * Get SPNEGO keytab Key from configuration.
- *
- * @param conf Configuration
- * @param defaultKey default key to be used for config lookup
- * @return DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY if the key is not empty
- * else return defaultKey
- */
- public static String getSpnegoKeytabKey(Configuration conf,
- String defaultKey) {
- String value =
- conf.get(
- DFSConfigKeysLegacy.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY);
- return (value == null || value.isEmpty()) ?
- defaultKey :
- DFSConfigKeysLegacy.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY;
- }
-
/**
* Leverages the Configuration.getPassword method to attempt to get
* passwords from the CredentialProvider API before falling back to
@@ -338,7 +360,7 @@
* @param alias name of the credential to retreive
* @return String credential value or null
*/
- static String getPassword(Configuration conf, String alias) {
+ static String getPassword(ConfigurationSource conf, String alias) {
String password = null;
try {
char[] passchars = conf.getPassword(alias);
@@ -356,8 +378,10 @@
/**
* Load HTTPS-related configuration.
*/
- public static Configuration loadSslConfiguration(Configuration conf) {
- Configuration sslConf = new Configuration(false);
+ public static ConfigurationSource loadSslConfiguration(
+ ConfigurationSource conf) {
+ Configuration sslConf =
+ new Configuration(false);
sslConf.addResource(conf.get(
OzoneConfigKeys.OZONE_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY,
@@ -382,7 +406,7 @@
boolean requireClientAuth = conf.getBoolean(
OZONE_CLIENT_HTTPS_NEED_AUTH_KEY, OZONE_CLIENT_HTTPS_NEED_AUTH_DEFAULT);
sslConf.setBoolean(OZONE_CLIENT_HTTPS_NEED_AUTH_KEY, requireClientAuth);
- return sslConf;
+ return new LegacyHadoopConfigurationSource(sslConf);
}
public InetSocketAddress getHttpAddress() {
@@ -413,4 +437,8 @@
protected abstract String getEnabledKey();
+ protected abstract String getHttpAuthType();
+
+ protected abstract String getHttpAuthConfigPrefix();
+
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/FilterContainer.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/FilterContainer.java
deleted file mode 100644
index 3b2b8eb..0000000
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/FilterContainer.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.server.http;
-
-import java.util.Map;
-
-/**
- * A container class for javax.servlet.Filter.
- */
-public interface FilterContainer {
- /**
- * Add a filter to the container.
- * @param name Filter name
- * @param classname Filter class name
- * @param parameters a map from parameter names to initial values
- */
- void addFilter(String name, String classname, Map<String, String> parameters);
-
- /**
- * Add a global filter to the container.
- * @param name filter name
- * @param classname filter class name
- * @param parameters a map from parameter names to initial values
- */
- void addGlobalFilter(String name, String classname,
- Map<String, String> parameters);
-}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/FilterInitializer.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/FilterInitializer.java
deleted file mode 100644
index 388fc21..0000000
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/FilterInitializer.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.server.http;
-
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * Initialize a javax.servlet.Filter.
- */
-public abstract class FilterInitializer {
- /**
- * Initialize a Filter to a FilterContainer.
- * @param container The filter container
- * @param conf Configuration for run-time parameters
- */
- public abstract void initFilter(FilterContainer container,
- Configuration conf);
-}
\ No newline at end of file
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpConfig.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpConfig.java
index 663a555..f340bdf 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpConfig.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpConfig.java
@@ -20,7 +20,7 @@
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.annotation.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.ozone.OzoneConfigKeys;
/**
@@ -61,7 +61,7 @@
}
}
- public static Policy getHttpPolicy(Configuration conf) {
+ public static Policy getHttpPolicy(ConfigurationSource conf) {
String policyStr = conf.get(OzoneConfigKeys.OZONE_HTTP_POLICY_KEY,
OzoneConfigKeys.OZONE_HTTP_POLICY_DEFAULT);
HttpConfig.Policy policy = HttpConfig.Policy.fromString(policyStr);
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpServer2.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpServer2.java
index 097580b..a37a08c 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpServer2.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpServer2.java
@@ -17,6 +17,17 @@
*/
package org.apache.hadoop.hdds.server.http;
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletContext;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletRequestWrapper;
+import javax.servlet.http.HttpServletResponse;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -37,29 +48,18 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import javax.servlet.Filter;
-import javax.servlet.FilterChain;
-import javax.servlet.FilterConfig;
-import javax.servlet.ServletContext;
-import javax.servlet.ServletException;
-import javax.servlet.ServletRequest;
-import javax.servlet.ServletResponse;
-import javax.servlet.http.HttpServlet;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletRequestWrapper;
-import javax.servlet.http.HttpServletResponse;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
-import com.sun.jersey.spi.container.servlet.ServletContainer;
import org.apache.hadoop.HadoopIllegalArgumentException;
-import org.apache.hadoop.hdds.annotation.InterfaceAudience;
-import org.apache.hadoop.hdds.annotation.InterfaceStability;
import org.apache.hadoop.conf.ConfServlet;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configuration.IntegerRanges;
import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.annotation.InterfaceStability;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
+import org.apache.hadoop.http.FilterContainer;
+import org.apache.hadoop.http.FilterInitializer;
+import org.apache.hadoop.http.lib.StaticUserWebFilter;
import org.apache.hadoop.jmx.JMXJsonServlet;
import org.apache.hadoop.log.LogLevel;
import org.apache.hadoop.security.AuthenticationFilterInitializer;
@@ -72,6 +72,11 @@
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.sun.jersey.spi.container.servlet.ServletContainer;
import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.server.ConnectionFactory;
import org.eclipse.jetty.server.Connector;
@@ -102,6 +107,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.security.AuthenticationFilterInitializer.getFilterConfigMap;
+
/**
* Create a Jetty embedded server to answer http requests. The primary goal is
* to serve up status information for the server. There are three contexts:
@@ -192,8 +199,8 @@
public static class Builder {
private ArrayList<URI> endpoints = Lists.newArrayList();
private String name;
- private Configuration conf;
- private Configuration sslConf;
+ private ConfigurationSource conf;
+ private ConfigurationSource sslConf;
private String[] pathSpecs;
private AccessControlList adminsAcl;
private boolean securityEnabled = false;
@@ -291,7 +298,7 @@
return this;
}
- public Builder setConf(Configuration configuration) {
+ public Builder setConf(ConfigurationSource configuration) {
this.conf = configuration;
return this;
}
@@ -300,7 +307,7 @@
* Specify the SSL configuration to load. This API provides an alternative
* to keyStore/keyPassword/trustStore.
*/
- public Builder setSSLConf(Configuration sslCnf) {
+ public Builder setSSLConf(ConfigurationSource sslCnf) {
this.sslConf = sslCnf;
return this;
}
@@ -368,14 +375,15 @@
}
/**
- * A wrapper of {@link Configuration#getPassword(String)}. It returns
+ * A wrapper of {@link ConfigurationSource#getPassword(String)}. It returns
* <code>String</code> instead of <code>char[]</code>.
*
* @param conf the configuration
* @param name the property name
* @return the password string or null
*/
- private static String getPasswordString(Configuration conf, String name)
+ private static String getPasswordString(ConfigurationSource conf,
+ String name)
throws IOException {
char[] passchars = conf.getPassword(name);
if (passchars == null) {
@@ -426,12 +434,14 @@
}
if (this.conf == null) {
- conf = new Configuration();
+ conf = new OzoneConfiguration();
}
HttpServer2 server = new HttpServer2(this);
if (this.securityEnabled) {
+ LOG.info("Initialize spnego with host: {} userKey: {} keytabKey: {}",
+ hostName, usernameConfKey, keytabConfKey);
server.initSpnego(conf, hostName, usernameConfKey, keytabConfKey);
}
@@ -561,12 +571,13 @@
this.findPort = b.findPort;
this.portRanges = b.portRanges;
- initializeWebServer(b.name, b.hostName, b.conf, b.pathSpecs);
+ initializeWebServer(b.name, b.hostName, b.conf, b.pathSpecs,
+ b.authFilterConfigurationPrefix);
}
private void initializeWebServer(String name, String hostName,
- Configuration conf, String[] pathSpecs)
- throws IOException {
+ ConfigurationSource conf, String[] pathSpecs,
+ String authFilterConfigPrefix) throws IOException {
Preconditions.checkNotNull(webAppContext);
@@ -602,10 +613,15 @@
addGlobalFilter("safety", QuotingInputFilter.class.getName(), xFrameParams);
final FilterInitializer[] initializers = getFilterInitializers(conf);
if (initializers != null) {
- conf = new Configuration(conf);
conf.set(BIND_ADDRESS, hostName);
for (FilterInitializer c : initializers) {
- c.initFilter(this, conf);
+ //c.initFilter(this, conf) does not work here as it does not take config
+ // prefix key.
+ Map<String, String> filterConfig = getFilterConfigMap(
+ LegacyHadoopConfigurationSource.asHadoopConfiguration(conf),
+ authFilterConfigPrefix);
+ addFilter("authentication", AuthenticationFilter.class.getName(),
+ filterConfig);
}
}
@@ -654,18 +670,20 @@
private static SignerSecretProvider constructSecretProvider(final Builder b,
ServletContext ctx)
throws Exception {
- final Configuration conf = b.conf;
+ final ConfigurationSource conf = b.conf;
Properties config = getFilterProperties(conf,
b.authFilterConfigurationPrefix);
return AuthenticationFilter.constructSecretProvider(
ctx, config, b.disallowFallbackToRandomSignerSecretProvider);
}
- private static Properties getFilterProperties(Configuration conf, String
+ private static Properties getFilterProperties(ConfigurationSource conf, String
prefix) {
Properties prop = new Properties();
Map<String, String> filterConfig = AuthenticationFilterInitializer
- .getFilterConfigMap(conf, prefix);
+ .getFilterConfigMap(
+ LegacyHadoopConfigurationSource.asHadoopConfiguration(conf),
+ prefix);
prop.putAll(filterConfig);
return prop;
}
@@ -678,7 +696,8 @@
/**
* Get an array of FilterConfiguration specified in the conf.
*/
- private static FilterInitializer[] getFilterInitializers(Configuration conf) {
+ private static FilterInitializer[] getFilterInitializers(
+ ConfigurationSource conf) {
if (conf == null) {
return null;
}
@@ -691,8 +710,12 @@
FilterInitializer[] initializers = new FilterInitializer[classes.length];
for (int i = 0; i < classes.length; i++) {
- initializers[i] = (FilterInitializer) ReflectionUtils.newInstance(
- classes[i], conf);
+ try {
+ initializers[i] = (FilterInitializer) classes[i].newInstance();
+ } catch (Exception e) {
+ LOG.error("Can't initialize the filter initializer {}",
+ classes[i].getCanonicalName(), e);
+ }
}
return initializers;
}
@@ -703,7 +726,7 @@
* @throws IOException
*/
protected void addDefaultApps(ContextHandlerCollection parent,
- final String appDir, Configuration conf) throws IOException {
+ final String appDir, ConfigurationSource conf) throws IOException {
// set up the context for "/logs/" if "hadoop.log.dir" property is defined
// and it's enabled.
String logDir = System.getProperty("hadoop.log.dir");
@@ -750,7 +773,7 @@
}
private void setContextAttributes(ServletContextHandler context,
- Configuration conf) {
+ ConfigurationSource conf) {
context.getServletContext().setAttribute(CONF_CONTEXT_ATTRIBUTE, conf);
context.getServletContext().setAttribute(ADMINS_ACL, adminsAcl);
}
@@ -1110,7 +1133,7 @@
pool.setMaxThreads(max);
}
- private void initSpnego(Configuration conf, String hostName,
+ private void initSpnego(ConfigurationSource conf, String hostName,
String usernameConfKey, String keytabConfKey) throws IOException {
Map<String, String> params = new HashMap<>();
String principalInConf = conf.get(usernameConfKey);
@@ -1380,8 +1403,9 @@
public static boolean isInstrumentationAccessAllowed(
ServletContext servletContext, HttpServletRequest request,
HttpServletResponse response) throws IOException {
- Configuration conf =
- (Configuration) servletContext.getAttribute(CONF_CONTEXT_ATTRIBUTE);
+ ConfigurationSource conf =
+ (ConfigurationSource) servletContext
+ .getAttribute(CONF_CONTEXT_ATTRIBUTE);
boolean access = true;
boolean adminAccess = conf.getBoolean(
@@ -1405,8 +1429,9 @@
public static boolean hasAdministratorAccess(
ServletContext servletContext, HttpServletRequest request,
HttpServletResponse response) throws IOException {
- Configuration conf =
- (Configuration) servletContext.getAttribute(CONF_CONTEXT_ATTRIBUTE);
+ ConfigurationSource conf =
+ (ConfigurationSource) servletContext
+ .getAttribute(CONF_CONTEXT_ATTRIBUTE);
// If there is no authorization, anybody has administrator access.
if (!conf.getBoolean(
CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, false)) {
@@ -1679,17 +1704,14 @@
}
}
- private Map<String, String> setHeaders(Configuration conf) {
+ private Map<String, String> setHeaders(ConfigurationSource conf) {
Map<String, String> xFrameParams = new HashMap<>();
- Map<String, String> headerConfigMap =
- conf.getValByRegex(HTTP_HEADER_REGEX);
xFrameParams.putAll(getDefaultHeaders());
if (this.xFrameOptionIsEnabled) {
xFrameParams.put(HTTP_HEADER_PREFIX + X_FRAME_OPTIONS,
this.xFrameOption.toString());
}
- xFrameParams.putAll(headerConfigMap);
return xFrameParams;
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/LogStreamServlet.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/LogStreamServlet.java
index d111547..b231154 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/LogStreamServlet.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/LogStreamServlet.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hdds.server.http;
-import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@@ -37,7 +36,7 @@
@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp)
- throws ServletException, IOException {
+ throws IOException {
WriterAppender appender =
new WriterAppender(new PatternLayout(PATTERN), resp.getWriter());
@@ -48,7 +47,7 @@
try {
Thread.sleep(Integer.MAX_VALUE);
} catch (InterruptedException e) {
- //interrupted
+ Thread.currentThread().interrupt();
}
} finally {
Logger.getRootLogger().removeAppender(appender);
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/PrometheusServlet.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/PrometheusServlet.java
index 77c2136..0d01aa4 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/PrometheusServlet.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/PrometheusServlet.java
@@ -34,6 +34,9 @@
*/
public class PrometheusServlet extends HttpServlet {
+ public static final String SECURITY_TOKEN = "PROMETHEUS_SECURITY_TOKEN";
+ public static final String BEARER = "Bearer";
+
public PrometheusMetricsSink getPrometheusSink() {
return
(PrometheusMetricsSink) getServletContext().getAttribute(
@@ -43,6 +46,18 @@
@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp)
throws ServletException, IOException {
+ String securityToken =
+ (String) getServletContext().getAttribute(SECURITY_TOKEN);
+ if (securityToken != null) {
+ String authorizationHeader = req.getHeader("Authorization");
+ if (authorizationHeader == null
+ || !authorizationHeader.startsWith(BEARER)
+ || !securityToken.equals(
+ authorizationHeader.substring(BEARER.length() + 1))) {
+ resp.setStatus(HttpServletResponse.SC_FORBIDDEN);
+ return;
+ }
+ }
DefaultMetricsSystem.instance().publishMetricsNow();
PrintWriter writer = resp.getWriter();
getPrometheusSink().writeMetrics(writer);
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/StaticUserWebFilter.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/StaticUserWebFilter.java
deleted file mode 100644
index c2d88cf..0000000
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/StaticUserWebFilter.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.server.http;
-
-import javax.servlet.Filter;
-import javax.servlet.FilterChain;
-import javax.servlet.FilterConfig;
-import javax.servlet.ServletException;
-import javax.servlet.ServletRequest;
-import javax.servlet.ServletResponse;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletRequestWrapper;
-import java.io.IOException;
-import java.security.Principal;
-import java.util.HashMap;
-
-import org.apache.hadoop.conf.Configuration;
-
-import static org.apache.hadoop.fs.CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER;
-import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Provides a servlet filter that pretends to authenticate a fake user (Dr.Who)
- * so that the web UI is usable for a secure cluster without authentication.
- */
-public class StaticUserWebFilter extends FilterInitializer {
- static final String DEPRECATED_UGI_KEY = "dfs.web.ugi";
-
- private static final Logger LOG =
- LoggerFactory.getLogger(StaticUserWebFilter.class);
-
- static class User implements Principal {
- private final String name;
-
- User(String name) {
- this.name = name;
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public int hashCode() {
- return name.hashCode();
- }
-
- @Override
- public boolean equals(Object other) {
- if (other == this) {
- return true;
- } else if (other == null || other.getClass() != getClass()) {
- return false;
- }
- return ((User) other).name.equals(name);
- }
-
- @Override
- public String toString() {
- return name;
- }
- }
-
- /**
- * JavaEE filter implementation to do the work.
- */
- public static class StaticUserFilter implements Filter {
- private User user;
- private String username;
-
- @Override
- public void destroy() {
- // NOTHING
- }
-
- @Override
- public void doFilter(ServletRequest request, ServletResponse response,
- FilterChain chain
- ) throws IOException, ServletException {
- HttpServletRequest httpRequest = (HttpServletRequest) request;
- // if the user is already authenticated, don't override it
- if (httpRequest.getRemoteUser() != null) {
- chain.doFilter(request, response);
- } else {
- HttpServletRequestWrapper wrapper =
- new HttpServletRequestWrapper(httpRequest) {
- @Override
- public Principal getUserPrincipal() {
- return user;
- }
-
- @Override
- public String getRemoteUser() {
- return username;
- }
- };
- chain.doFilter(wrapper, response);
- }
- }
-
- @Override
- public void init(FilterConfig conf) throws ServletException {
- this.username = conf.getInitParameter(HADOOP_HTTP_STATIC_USER);
- this.user = new User(username);
- }
-
- }
-
- @Override
- public void initFilter(FilterContainer container, Configuration conf) {
- HashMap<String, String> options = new HashMap<String, String>();
-
- String username = getUsernameFromConf(conf);
- options.put(HADOOP_HTTP_STATIC_USER, username);
-
- container.addFilter("static_user_filter",
- StaticUserFilter.class.getName(),
- options);
- }
-
- /**
- * Retrieve the static username from the configuration.
- */
- static String getUsernameFromConf(Configuration conf) {
- String oldStyleUgi = conf.get(DEPRECATED_UGI_KEY);
- if (oldStyleUgi != null) {
- // We can't use the normal configuration deprecation mechanism here
- // since we need to split out the username from the configured UGI.
- LOG.warn(DEPRECATED_UGI_KEY + " should not be used. Instead, use " +
- HADOOP_HTTP_STATIC_USER + ".");
- String[] parts = oldStyleUgi.split(",");
- return parts[0];
- } else {
- return conf.get(HADOOP_HTTP_STATIC_USER,
- DEFAULT_HADOOP_HTTP_STATIC_USER);
- }
- }
-
-}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java
index 17fa392..214adb2 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java
@@ -24,8 +24,8 @@
import java.util.OptionalInt;
import java.util.concurrent.TimeUnit;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.DFSConfigKeysLegacy;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
@@ -87,7 +87,7 @@
* @return Target {@code InetSocketAddress} for the SCM service endpoint.
*/
public static InetSocketAddress getScmAddressForDataNodes(
- Configuration conf) {
+ ConfigurationSource conf) {
// We try the following settings in decreasing priority to retrieve the
// target host.
// - OZONE_SCM_DATANODE_ADDRESS_KEY
@@ -118,7 +118,7 @@
* @return Target {@code InetSocketAddress} for the SCM client endpoint.
*/
public static InetSocketAddress getScmClientBindAddress(
- Configuration conf) {
+ ConfigurationSource conf) {
final String host = getHostNameFromConfigKeys(conf,
ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_KEY)
.orElse(ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_DEFAULT);
@@ -138,7 +138,7 @@
* @return Target {@code InetSocketAddress} for the SCM block client endpoint.
*/
public static InetSocketAddress getScmBlockClientBindAddress(
- Configuration conf) {
+ ConfigurationSource conf) {
final String host = getHostNameFromConfigKeys(conf,
ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_BIND_HOST_KEY)
.orElse(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_BIND_HOST_DEFAULT);
@@ -158,7 +158,7 @@
* @return Target {@code InetSocketAddress} for the SCM security service.
*/
public static InetSocketAddress getScmSecurityInetAddress(
- Configuration conf) {
+ ConfigurationSource conf) {
final String host = getHostNameFromConfigKeys(conf,
ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_BIND_HOST_KEY)
.orElse(ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_BIND_HOST_DEFAULT);
@@ -182,7 +182,7 @@
* @return Target {@code InetSocketAddress} for the SCM service endpoint.
*/
public static InetSocketAddress getScmDataNodeBindAddress(
- Configuration conf) {
+ ConfigurationSource conf) {
final Optional<String> host = getHostNameFromConfigKeys(conf,
ScmConfigKeys.OZONE_SCM_DATANODE_BIND_HOST_KEY);
@@ -203,7 +203,7 @@
* @return Target {@code InetSocketAddress} for the SCM service endpoint.
*/
public static InetSocketAddress getReconDataNodeBindAddress(
- Configuration conf) {
+ ConfigurationSource conf) {
final Optional<String> host = getHostNameFromConfigKeys(conf,
ReconConfigKeys.OZONE_RECON_DATANODE_BIND_HOST_KEY);
@@ -222,7 +222,7 @@
* @param conf - Configuration
* @return long in Milliseconds.
*/
- public static long getScmheartbeatCheckerInterval(Configuration conf) {
+ public static long getScmheartbeatCheckerInterval(ConfigurationSource conf) {
return conf.getTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL,
ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_DEFAULT,
TimeUnit.MILLISECONDS);
@@ -235,7 +235,7 @@
* @param conf - Ozone Config
* @return - HB interval in milli seconds.
*/
- public static long getScmHeartbeatInterval(Configuration conf) {
+ public static long getScmHeartbeatInterval(ConfigurationSource conf) {
return conf.getTimeDuration(HDDS_HEARTBEAT_INTERVAL,
HDDS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS);
}
@@ -247,7 +247,7 @@
* @param conf - Configuration.
* @return - Long, Milliseconds to wait before flagging a node as stale.
*/
- public static long getStaleNodeInterval(Configuration conf) {
+ public static long getStaleNodeInterval(ConfigurationSource conf) {
long staleNodeIntervalMs =
conf.getTimeDuration(OZONE_SCM_STALENODE_INTERVAL,
@@ -284,7 +284,7 @@
* @param conf - Configuration.
* @return - the interval for dead node flagging.
*/
- public static long getDeadNodeInterval(Configuration conf) {
+ public static long getDeadNodeInterval(ConfigurationSource conf) {
long staleNodeIntervalMs = getStaleNodeInterval(conf);
long deadNodeIntervalMs = conf.getTimeDuration(OZONE_SCM_DEADNODE_INTERVAL,
OZONE_SCM_DEADNODE_INTERVAL_DEFAULT,
@@ -303,7 +303,7 @@
* @param conf - Ozone Config
* @return - Rpc timeout in Milliseconds.
*/
- public static long getScmRpcTimeOutInMilliseconds(Configuration conf) {
+ public static long getScmRpcTimeOutInMilliseconds(ConfigurationSource conf) {
return conf.getTimeDuration(OZONE_SCM_HEARTBEAT_RPC_TIMEOUT,
OZONE_SCM_HEARTBEAT_RPC_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS);
}
@@ -314,7 +314,7 @@
* @param conf - Ozone Config
* @return - Log warn interval.
*/
- public static int getLogWarnInterval(Configuration conf) {
+ public static int getLogWarnInterval(ConfigurationSource conf) {
return conf.getInt(OZONE_SCM_HEARTBEAT_LOG_WARN_INTERVAL_COUNT,
OZONE_SCM_HEARTBEAT_LOG_WARN_DEFAULT);
}
@@ -324,14 +324,15 @@
* @param conf - Conf
* @return port number.
*/
- public static int getContainerPort(Configuration conf) {
+ public static int getContainerPort(ConfigurationSource conf) {
return conf.getInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
}
- public static String getOzoneDatanodeRatisDirectory(Configuration conf) {
+ public static String getOzoneDatanodeRatisDirectory(
+ ConfigurationSource conf) {
String storageDir = conf.get(
- OzoneConfigKeys.DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR);
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR);
if (Strings.isNullOrEmpty(storageDir)) {
storageDir = ServerUtils.getDefaultRatisDirectory(conf);
@@ -339,15 +340,13 @@
return storageDir;
}
-
-
/**
* Get the path for datanode id file.
*
* @param conf - Configuration
* @return the path of datanode id as string
*/
- public static String getDatanodeIdFilePath(Configuration conf) {
+ public static String getDatanodeIdFilePath(ConfigurationSource conf) {
String dataNodeIDDirPath =
conf.get(ScmConfigKeys.OZONE_SCM_DATANODE_ID_DIR);
if (dataNodeIDDirPath == null) {
@@ -404,7 +403,7 @@
* @throws IllegalArgumentException if configuration is not defined or invalid
*/
public static InetSocketAddress getScmAddressForSecurityProtocol(
- Configuration conf) {
+ ConfigurationSource conf) {
Optional<String> host = getHostNameFromConfigKeys(conf,
ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY,
ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/MetadataKeyFilters.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/MetadataKeyFilters.java
index 9fcc270..371cef6 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/MetadataKeyFilters.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/MetadataKeyFilters.java
@@ -44,7 +44,10 @@
.addFilter(OzoneConsts.DELETING_KEY_PREFIX, true)
.addFilter(OzoneConsts.DELETED_KEY_PREFIX, true)
.addFilter(OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX, true)
- .addFilter(OzoneConsts.BLOCK_COMMIT_SEQUENCE_ID_PREFIX, true);
+ .addFilter(OzoneConsts.BLOCK_COMMIT_SEQUENCE_ID_PREFIX, true)
+ .addFilter(OzoneConsts.BLOCK_COUNT, true)
+ .addFilter(OzoneConsts.CONTAINER_BYTES_USED, true)
+ .addFilter(OzoneConsts.PENDING_DELETE_BLOCK_COUNT, true);
private MetadataKeyFilters() {
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/MetadataStoreBuilder.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/MetadataStoreBuilder.java
index dafa92b..d697fdf 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/MetadataStoreBuilder.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/MetadataStoreBuilder.java
@@ -24,7 +24,7 @@
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConfigKeys;
@@ -34,7 +34,6 @@
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS_OFF;
-
import org.iq80.leveldb.Options;
import org.rocksdb.BlockBasedTableConfig;
import org.rocksdb.Statistics;
@@ -52,11 +51,11 @@
private File dbFile;
private long cacheSize;
private boolean createIfMissing = true;
- private Optional<Configuration> optionalConf = Optional.empty();
+ private Optional<ConfigurationSource> optionalConf = Optional.empty();
private String dbType;
@VisibleForTesting
- public static final Map<Configuration, org.rocksdb.Options> CACHED_OPTS =
- new ConcurrentHashMap<>();
+ public static final Map<ConfigurationSource, org.rocksdb.Options>
+ CACHED_OPTS = new ConcurrentHashMap<>();
@VisibleForTesting
public static final OzoneConfiguration DEFAULT_CONF =
new OzoneConfiguration();
@@ -80,7 +79,7 @@
return this;
}
- public MetadataStoreBuilder setConf(Configuration configuration) {
+ public MetadataStoreBuilder setConf(ConfigurationSource configuration) {
this.optionalConf = Optional.of(configuration);
return this;
}
@@ -102,7 +101,7 @@
}
// Build db store based on configuration
- final Configuration conf = optionalConf.orElse(DEFAULT_CONF);
+ final ConfigurationSource conf = optionalConf.orElse(DEFAULT_CONF);
if (dbType == null) {
LOG.debug("dbType is null, using ");
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/ProtocolMessageMetrics.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/ProtocolMessageMetrics.java
index 73c3f01..aa50eab 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/ProtocolMessageMetrics.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/ProtocolMessageMetrics.java
@@ -25,41 +25,43 @@
import org.apache.hadoop.metrics2.MetricsInfo;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.MetricsSource;
+import org.apache.hadoop.metrics2.MetricsTag;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-
-import com.google.protobuf.ProtocolMessageEnum;
+import org.apache.hadoop.metrics2.lib.Interns;
/**
* Metrics to count all the subtypes of a specific message.
*/
-public class ProtocolMessageMetrics implements MetricsSource {
+public class ProtocolMessageMetrics<KEY> implements MetricsSource {
private String name;
private String description;
- private Map<ProtocolMessageEnum, AtomicLong> counters =
+ private Map<KEY, AtomicLong> counters =
new ConcurrentHashMap<>();
- public static ProtocolMessageMetrics create(String name,
- String description, ProtocolMessageEnum[] types) {
- ProtocolMessageMetrics protocolMessageMetrics =
- new ProtocolMessageMetrics(name, description,
- types);
- return protocolMessageMetrics;
+ private Map<KEY, AtomicLong> elapsedTimes =
+ new ConcurrentHashMap<>();
+
+ public static <KEY> ProtocolMessageMetrics<KEY> create(String name,
+ String description, KEY[] types) {
+ return new ProtocolMessageMetrics<KEY>(name, description, types);
}
public ProtocolMessageMetrics(String name, String description,
- ProtocolMessageEnum[] values) {
+ KEY[] values) {
this.name = name;
this.description = description;
- for (ProtocolMessageEnum value : values) {
+ for (KEY value : values) {
counters.put(value, new AtomicLong(0));
+ elapsedTimes.put(value, new AtomicLong(0));
}
}
- public void increment(ProtocolMessageEnum key) {
+ public void increment(KEY key, long duration) {
counters.get(key).incrementAndGet();
+ elapsedTimes.get(key).addAndGet(duration);
}
public void register() {
@@ -73,11 +75,19 @@
@Override
public void getMetrics(MetricsCollector collector, boolean all) {
- MetricsRecordBuilder builder = collector.addRecord(name);
counters.forEach((key, value) -> {
- builder.addCounter(new MetricName(key.toString(), ""), value.longValue());
+ MetricsRecordBuilder builder =
+ collector.addRecord(name);
+ builder.add(
+ new MetricsTag(Interns.info("type", "Message type"), key.toString()));
+ builder.addCounter(new MetricName("counter", "Number of distinct calls"),
+ value.longValue());
+ builder.addCounter(
+ new MetricName("time", "Sum of the duration of the calls"),
+ elapsedTimes.get(key).longValue());
+ builder.endRecord();
+
});
- builder.endRecord();
}
/**
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/BatchOperationHandler.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/BatchOperationHandler.java
new file mode 100644
index 0000000..eea483c
--- /dev/null
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/BatchOperationHandler.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.utils.db;
+
+import java.io.IOException;
+
+/**
+ * Create and commit batch operation for one DB.
+ */
+public interface BatchOperationHandler {
+
+ /**
+ * Initialize an atomic batch operation which can hold multiple PUT/DELETE
+ * operations and committed later in one step.
+ *
+ * @return BatchOperation holder which can be used to add or commit batch
+ * operations.
+ */
+ BatchOperation initBatchOperation();
+
+ /**
+ * Commit the batch operations.
+ *
+ * @param operation which contains all the required batch operation.
+ * @throws IOException on Failure.
+ */
+ void commitBatchOperation(BatchOperation operation) throws IOException;
+}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBColumnFamilyDefinition.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBColumnFamilyDefinition.java
new file mode 100644
index 0000000..e1c4163
--- /dev/null
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBColumnFamilyDefinition.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.utils.db;
+
+import java.io.IOException;
+
+/**
+ * Class represents one single column table with the required codecs and types.
+ *
+ * @param <KEY> the type of the key.
+ * @param <VALUE> they type of the value.
+ */
+public class DBColumnFamilyDefinition<KEY, VALUE> {
+
+ private final String tableName;
+
+ private final Class<KEY> keyType;
+
+ private final Codec<KEY> keyCodec;
+
+ private final Class<VALUE> valueType;
+
+ private final Codec<VALUE> valueCodec;
+
+ public DBColumnFamilyDefinition(
+ String tableName,
+ Class<KEY> keyType,
+ Codec<KEY> keyCodec,
+ Class<VALUE> valueType,
+ Codec<VALUE> valueCodec) {
+ this.tableName = tableName;
+ this.keyType = keyType;
+ this.keyCodec = keyCodec;
+ this.valueType = valueType;
+ this.valueCodec = valueCodec;
+ }
+
+ public Table<KEY, VALUE> getTable(DBStore db) throws IOException {
+ return db.getTable(tableName, keyType, valueType);
+ }
+
+ public String getName() {
+ return tableName;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public Class<KEY> getKeyType() {
+ return keyType;
+ }
+
+ public Codec<KEY> getKeyCodec() {
+ return keyCodec;
+ }
+
+ public Class<VALUE> getValueType() {
+ return valueType;
+ }
+
+ public Codec<VALUE> getValueCodec() {
+ return valueCodec;
+ }
+}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBDefinition.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBDefinition.java
new file mode 100644
index 0000000..3058261
--- /dev/null
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBDefinition.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.utils.db;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Simple interface to provide information to create a DBStore..
+ */
+public interface DBDefinition {
+
+ Logger LOG = LoggerFactory.getLogger(DBDefinition.class);
+
+ /**
+ * Logical name of the DB.
+ */
+ String getName();
+
+ /**
+ * Configuration key defines the location of the DB.
+ */
+ String getLocationConfigKey();
+
+ /**
+ * Create a new DB store instance based on the configuration.
+ */
+ DBColumnFamilyDefinition[] getColumnFamilies();
+
+}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java
index ed64b74..8567d03 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java
@@ -35,7 +35,7 @@
*
*/
@InterfaceStability.Evolving
-public interface DBStore extends AutoCloseable {
+public interface DBStore extends AutoCloseable, BatchOperationHandler {
/**
* Gets an existing TableStore.
@@ -141,22 +141,6 @@
*/
long getEstimatedKeyCount() throws IOException;
- /**
- * Initialize an atomic batch operation which can hold multiple PUT/DELETE
- * operations and committed later in one step.
- *
- * @return BatchOperation holder which can be used to add or commit batch
- * operations.
- */
- BatchOperation initBatchOperation();
-
- /**
- * Commit the batch operations.
- *
- * @param operation which contains all the required batch operation.
- * @throws IOException on Failure.
- */
- void commitBatchOperation(BatchOperation operation) throws IOException;
/**
* Get current snapshot of DB store as an artifact stored on
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java
index c9906d1..2e18530 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java
@@ -29,12 +29,15 @@
import java.util.List;
import java.util.Set;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.StringUtils;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import com.google.common.base.Preconditions;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DB_PROFILE;
+import static org.apache.hadoop.hdds.server.ServerUtils.getDirectoryFromConfig;
+import static org.apache.hadoop.hdds.server.ServerUtils.getOzoneMetaDirPath;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS_OFF;
@@ -73,15 +76,16 @@
private String dbname;
private Path dbPath;
private List<String> tableNames;
- private Configuration configuration;
+ private ConfigurationSource configuration;
private CodecRegistry registry;
private String rocksDbStat;
private RocksDBConfiguration rocksDBConfiguration;
- private DBStoreBuilder(OzoneConfiguration configuration) {
+ private DBStoreBuilder(ConfigurationSource configuration) {
this(configuration, configuration.getObject(RocksDBConfiguration.class));
}
- private DBStoreBuilder(OzoneConfiguration configuration,
+
+ private DBStoreBuilder(ConfigurationSource configuration,
RocksDBConfiguration rocksDBConfiguration) {
tables = new HashSet<>();
tableNames = new LinkedList<>();
@@ -93,8 +97,7 @@
this.rocksDBConfiguration = rocksDBConfiguration;
}
-
- public static DBStoreBuilder newBuilder(OzoneConfiguration configuration) {
+ public static DBStoreBuilder newBuilder(ConfigurationSource configuration) {
return new DBStoreBuilder(configuration);
}
@@ -263,4 +266,45 @@
return Paths.get(dbPath.toString(), dbname).toFile();
}
+ private static DBStoreBuilder createDBStoreBuilder(
+ ConfigurationSource configuration, DBDefinition definition) {
+
+ File metadataDir = getDirectoryFromConfig(configuration,
+ definition.getLocationConfigKey(), definition.getName());
+
+ if (metadataDir == null) {
+
+ LOG.warn("{} is not configured. We recommend adding this setting. " +
+ "Falling back to {} instead.",
+ definition.getLocationConfigKey(),
+ HddsConfigKeys.OZONE_METADATA_DIRS);
+ metadataDir = getOzoneMetaDirPath(configuration);
+ }
+
+ return DBStoreBuilder.newBuilder(configuration)
+ .setName(definition.getName())
+ .setPath(Paths.get(metadataDir.getPath()));
+ }
+
+ /**
+ * Create DBStoreBuilder from a generic DBDefinition.
+ */
+ public static DBStore createDBStore(ConfigurationSource configuration,
+ DBDefinition definition)
+ throws IOException {
+ DBStoreBuilder builder = createDBStoreBuilder(configuration, definition);
+ for (DBColumnFamilyDefinition columnTableDefinition : definition
+ .getColumnFamilies()) {
+ builder.registerTable(columnTableDefinition);
+ }
+ return builder.build();
+ }
+
+ private <KEY, VALUE> void registerTable(
+ DBColumnFamilyDefinition<KEY, VALUE> definition) {
+ addTable(definition.getName())
+ .addCodec(definition.getKeyType(), definition.getKeyCodec())
+ .addCodec(definition.getValueType(), definition.getValueCodec());
+ }
+
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBMetrics.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBMetrics.java
index 84597aa..d5d1f36 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBMetrics.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBMetrics.java
@@ -34,14 +34,20 @@
private static final String SOURCE_NAME =
RDBMetrics.class.getSimpleName();
+ private static RDBMetrics instance;
+
public RDBMetrics() {
}
- public static RDBMetrics create() {
+ public static synchronized RDBMetrics create() {
+ if (instance != null) {
+ return instance;
+ }
MetricsSystem ms = DefaultMetricsSystem.instance();
- return ms.register(SOURCE_NAME,
+ instance = ms.register(SOURCE_NAME,
"Rocks DB Metrics",
new RDBMetrics());
+ return instance;
}
private @Metric MutableCounterLong numDBKeyMayExistChecks;
@@ -95,7 +101,8 @@
return numDBKeyMayExistMisses.value();
}
- public void unRegister() {
+ public static void unRegister() {
+ instance = null;
MetricsSystem ms = DefaultMetricsSystem.instance();
ms.unregisterSource(SOURCE_NAME);
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
index 4bb9431..20faf24 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
@@ -182,7 +182,7 @@
statMBeanName = null;
}
- rdbMetrics.unRegister();
+ RDBMetrics.unRegister();
if (db != null) {
db.close();
}
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/TestJsonUtils.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/TestJsonUtils.java
new file mode 100644
index 0000000..a9f82ed
--- /dev/null
+++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/TestJsonUtils.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.server;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdds.client.OzoneQuota;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test the json object printer.
+ */
+public class TestJsonUtils {
+
+ @Test
+ public void printObjectAsJson() throws IOException {
+ OzoneQuota quota = new OzoneQuota(123, OzoneQuota.Units.MB);
+
+ String result = JsonUtils.toJsonStringWithDefaultPrettyPrinter(quota);
+
+ assertContains(result, "\"size\" : 123");
+ assertContains(result, "\"unit\" : \"MB\"");
+ }
+
+ private static void assertContains(String str, String part) {
+ assertTrue("Expected JSON to contain '" + part + "', but didn't: " + str,
+ str.contains(part));
+ }
+}
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/TestServerUtils.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/TestServerUtils.java
index 9735d2c..1ebde84 100644
--- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/TestServerUtils.java
+++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/TestServerUtils.java
@@ -17,21 +17,20 @@
*/
package org.apache.hadoop.hdds.server;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
+import java.io.File;
+
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.test.PathUtils;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import java.io.File;
-
+import org.apache.commons.io.FileUtils;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
/**
* Unit tests for {@link ServerUtils}.
@@ -49,7 +48,7 @@
final File testDir = PathUtils.getTestDir(TestServerUtils.class);
final File dbDir = new File(testDir, "scmDbDir");
final File metaDir = new File(testDir, "metaDir");
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
conf.set(ScmConfigKeys.OZONE_SCM_DB_DIRS, dbDir.getPath());
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.getPath());
@@ -72,7 +71,7 @@
public void testGetScmDbDirWithFallback() {
final File testDir = PathUtils.getTestDir(TestServerUtils.class);
final File metaDir = new File(testDir, "metaDir");
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.getPath());
try {
assertFalse(metaDir.exists());
@@ -99,7 +98,7 @@
public void ozoneMetadataDirAcceptsSingleItem() {
final File testDir = PathUtils.getTestDir(TestServerUtils.class);
final File metaDir = new File(testDir, "metaDir");
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.getPath());
try {
@@ -113,7 +112,7 @@
@Test
public void ozoneMetadataDirRejectsList() {
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, "/data/meta1,/data/meta2");
thrown.expect(IllegalArgumentException.class);
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestBaseHttpServer.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestBaseHttpServer.java
index d1832e2..4f73375 100644
--- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestBaseHttpServer.java
+++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestBaseHttpServer.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.hdds.server.http;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.junit.Assert;
import org.junit.Test;
@@ -28,7 +28,7 @@
public class TestBaseHttpServer {
@Test
public void getBindAddress() throws Exception {
- Configuration conf = new Configuration();
+ OzoneConfiguration conf = new OzoneConfiguration();
conf.set("enabled", "false");
BaseHttpServer baseHttpServer = new BaseHttpServer(conf, "test") {
@@ -81,6 +81,16 @@
protected String getEnabledKey() {
return "enabled";
}
+
+ @Override
+ protected String getHttpAuthType() {
+ return "simple";
+ }
+
+ @Override
+ protected String getHttpAuthConfigPrefix() {
+ return null;
+ }
};
conf.set("addresskey", "0.0.0.0:1234");
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestMetadataStore.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestMetadataStore.java
index bc53c7a..3eb832f 100644
--- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestMetadataStore.java
+++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestMetadataStore.java
@@ -28,7 +28,6 @@
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.StringUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.utils.MetadataKeyFilters.KeyPrefixFilter;
@@ -90,7 +89,7 @@
testDir = GenericTestUtils.getTestDir(getClass().getSimpleName()
+ "-" + storeImpl.toLowerCase());
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl);
store = MetadataStoreBuilder.newBuilder()
@@ -110,7 +109,7 @@
@Test
public void testIterator() throws Exception {
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl);
File dbDir = GenericTestUtils.getRandomizedTestDir();
MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
@@ -166,7 +165,7 @@
@Test
public void testMetaStoreConfigDifferentFromType() throws IOException {
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl);
String dbType;
GenericTestUtils.setLogLevel(MetadataStoreBuilder.LOG, Level.DEBUG);
@@ -193,7 +192,7 @@
@Test
public void testdbTypeNotSet() throws IOException {
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl);
GenericTestUtils.setLogLevel(MetadataStoreBuilder.LOG, Level.DEBUG);
GenericTestUtils.LogCapturer logCapturer =
@@ -460,7 +459,7 @@
@Test
public void testDestroyDB() throws IOException {
// create a new DB to test db destroy
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl);
File dbDir = GenericTestUtils.getTestDir(getClass().getSimpleName()
@@ -485,7 +484,7 @@
@Test
public void testBatchWrite() throws IOException {
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl);
File dbDir = GenericTestUtils.getTestDir(getClass().getSimpleName()
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestRocksDBStoreMBean.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestRocksDBStoreMBean.java
index 29c7803..610e898 100644
--- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestRocksDBStoreMBean.java
+++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestRocksDBStoreMBean.java
@@ -17,7 +17,13 @@
*/
package org.apache.hadoop.hdds.utils;
-import org.apache.hadoop.conf.Configuration;
+import javax.management.MBeanServer;
+import java.io.File;
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.util.HashMap;
+import java.util.Map;
+
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.metrics2.AbstractMetric;
import org.apache.hadoop.metrics2.MetricsCollector;
@@ -29,27 +35,20 @@
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import javax.management.MBeanServer;
-import java.io.File;
-import java.io.IOException;
-import java.lang.management.ManagementFactory;
-import java.util.HashMap;
-import java.util.Map;
import static java.nio.charset.StandardCharsets.UTF_8;
+import org.junit.Assert;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import org.junit.Before;
+import org.junit.Test;
/**
* Test the JMX interface for the rocksdb metastore implementation.
*/
public class TestRocksDBStoreMBean {
- private Configuration conf;
+ private OzoneConfiguration conf;
@Before
public void init() throws Exception {
diff --git a/hadoop-hdds/hadoop-dependency-client/README.md b/hadoop-hdds/hadoop-dependency-client/README.md
new file mode 100644
index 0000000..0ca9a1c
--- /dev/null
+++ b/hadoop-hdds/hadoop-dependency-client/README.md
@@ -0,0 +1,60 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+This helper project provides a reduced view to the Hadoop dependencies. Hadoop common by default depends on a lot of other projects
+which are not required for the parts used in Ozone.
+
+The exclude rules should be updated on each version bump. If Hadoop introduce new dependencies we need to exclude them (unless they are required).
+
+
+For the reference here are the minimal jar files wich are required by Ozone:
+
+
+```
+mvn dependency:tree
+
+[INFO] Scanning for projects...
+[INFO]
+[INFO] -------< org.apache.hadoop:hadoop-hdds-hadoop-dependency-client >-------
+[INFO] Building Apache Hadoop HDDS Hadoop Client dependencies 0.6.0-SNAPSHOT
+[INFO] --------------------------------[ jar ]---------------------------------
+[INFO]
+[INFO] --- maven-dependency-plugin:3.0.2:tree (default-cli) @ hadoop-hdds-hadoop-dependency-client ---
+[INFO] org.apache.hadoop:hadoop-hdds-hadoop-dependency-client:jar:0.6.0-SNAPSHOT
+[INFO] +- org.apache.hadoop:hadoop-annotations:jar:3.2.0:compile
+[INFO] | \- jdk.tools:jdk.tools:jar:1.8:system
+[INFO] +- org.apache.hadoop:hadoop-common:jar:3.2.0:compile
+[INFO] | +- org.apache.httpcomponents:httpclient:jar:4.5.2:compile
+[INFO] | | \- org.apache.httpcomponents:httpcore:jar:4.4.4:compile
+[INFO] | +- org.apache.commons:commons-configuration2:jar:2.1.1:compile
+[INFO] | +- com.google.re2j:re2j:jar:1.1:compile
+[INFO] | +- com.google.protobuf:protobuf-java:jar:2.5.0:compile
+[INFO] | +- org.apache.hadoop:hadoop-auth:jar:3.2.0:compile
+[INFO] | +- com.google.code.findbugs:jsr305:jar:3.0.0:compile
+[INFO] | +- org.apache.htrace:htrace-core4:jar:4.1.0-incubating:compile
+[INFO] | +- org.codehaus.woodstox:stax2-api:jar:3.1.4:compile
+[INFO] | \- com.fasterxml.woodstox:woodstox-core:jar:5.0.3:compile
+[INFO] +- org.apache.hadoop:hadoop-hdfs:jar:3.2.0:compile
+[INFO] \- junit:junit:jar:4.11:test
+[INFO] \- org.hamcrest:hamcrest-core:jar:1.3:test
+[INFO] ------------------------------------------------------------------------
+[INFO] BUILD SUCCESS
+[INFO] ------------------------------------------------------------------------
+[INFO] Total time: 1.144 s
+[INFO] Finished at: 2020-04-01T11:21:46+02:00
+[INFO] ------------------------------------------------------------------------
+```
diff --git a/hadoop-hdds/hadoop-dependency-client/pom.xml b/hadoop-hdds/hadoop-dependency-client/pom.xml
new file mode 100644
index 0000000..9a735df
--- /dev/null
+++ b/hadoop-hdds/hadoop-dependency-client/pom.xml
@@ -0,0 +1,282 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds</artifactId>
+ <version>0.6.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>hadoop-hdds-hadoop-dependency-client</artifactId>
+ <version>0.6.0-SNAPSHOT</version>
+ <description>Apache Hadoop Distributed Data Store Hadoop client dependencies
+ </description>
+ <name>Apache Hadoop HDDS Hadoop Client dependencies</name>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math3</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-net</groupId>
+ <artifactId>commons-net</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-collections</groupId>
+ <artifactId>commons-collections</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-webapp</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-servlet</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-text</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.jcraft</groupId>
+ <artifactId>jsch</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.kerby</groupId>
+ <artifactId>kerb-simplekdc</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>dnsjava</groupId>
+ <artifactId>dnsjava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.nimbusds</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>net.minidev</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-util-ajax</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-daemon</groupId>
+ <artifactId>commons-daemon</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.htrace</groupId>
+ <artifactId>htrace-core4</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.fusesource.leveldbjni</groupId>
+ <artifactId>leveldbjni-all</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.fusesource.leveldbjni</groupId>
+ <artifactId>leveldbjni-all</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hadoop-hdds/hadoop-dependency-server/pom.xml b/hadoop-hdds/hadoop-dependency-server/pom.xml
new file mode 100644
index 0000000..e20b554
--- /dev/null
+++ b/hadoop-hdds/hadoop-dependency-server/pom.xml
@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds</artifactId>
+ <version>0.6.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>hadoop-hdds-hadoop-dependency-server</artifactId>
+ <version>0.6.0-SNAPSHOT</version>
+ <description>Apache Hadoop Distributed Data Store Hadoop server dependencies
+ </description>
+ <name>Apache Hadoop HDDS Hadoop Server dependencies</name>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-core-asl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-jaxrs</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-xc</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <!-- commons-cli is required by DFSUtil.addPBProtocol -->
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hadoop-hdds/hadoop-dependency-test/pom.xml b/hadoop-hdds/hadoop-dependency-test/pom.xml
new file mode 100644
index 0000000..aae0f75
--- /dev/null
+++ b/hadoop-hdds/hadoop-dependency-test/pom.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds</artifactId>
+ <version>0.6.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>hadoop-hdds-hadoop-dependency-test</artifactId>
+ <version>0.6.0-SNAPSHOT</version>
+ <description>Apache Hadoop Distributed Data Store Hadoop test dependencies
+ </description>
+ <name>Apache Hadoop HDDS Hadoop Test dependencies</name>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ <type>test-jar</type>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hadoop-hdds/pom.xml b/hadoop-hdds/pom.xml
index 87a6f30..92c3f39 100644
--- a/hadoop-hdds/pom.xml
+++ b/hadoop-hdds/pom.xml
@@ -30,6 +30,9 @@
<packaging>pom</packaging>
<modules>
+ <module>hadoop-dependency-client</module>
+ <module>hadoop-dependency-test</module>
+ <module>hadoop-dependency-server</module>
<module>client</module>
<module>common</module>
<module>framework</module>
@@ -38,16 +41,9 @@
<module>tools</module>
<module>docs</module>
<module>config</module>
+ <module>test-utils</module>
</modules>
- <properties>
- <protobuf-maven-plugin.version>0.5.1</protobuf-maven-plugin.version>
- <os-maven-plugin.version>1.5.0.Final</os-maven-plugin.version>
-
- <maven-surefire-plugin.version>3.0.0-M1</maven-surefire-plugin.version>
-
- <junit.jupiter.version>5.3.1</junit.jupiter.version>
- </properties>
<repositories>
<repository>
<id>apache.snapshots.https</id>
@@ -72,6 +68,25 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-hadoop-dependency-client</artifactId>
+ <version>${hdds.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-hadoop-dependency-server</artifactId>
+ <version>${hdds.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-hadoop-dependency-test</artifactId>
+ <version>${hdds.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-client</artifactId>
<version>${hdds.version}</version>
</dependency>
@@ -114,6 +129,13 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-test-utils</artifactId>
+ <version>${hdds.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-common</artifactId>
<version>${hdds.version}</version>
<type>test-jar</type>
@@ -134,18 +156,6 @@
</dependency>
<dependency>
- <groupId>org.openjdk.jmh</groupId>
- <artifactId>jmh-core</artifactId>
- <version>1.19</version>
- </dependency>
-
- <dependency>
- <groupId>org.openjdk.jmh</groupId>
- <artifactId>jmh-generator-annprocess</artifactId>
- <version>1.19</version>
- </dependency>
-
- <dependency>
<groupId>org.apache.ratis</groupId>
<artifactId>ratis-proto-shaded</artifactId>
<version>${ratis.version}</version>
@@ -186,13 +196,6 @@
<artifactId>bcpkix-jdk15on</artifactId>
<version>${bouncycastle.version}</version>
</dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-api</artifactId>
- <version>${junit.jupiter.version}</version>
- <scope>test</scope>
- </dependency>
</dependencies>
</dependencyManagement>
<dependencies>
@@ -245,6 +248,7 @@
<exclude>src/test/resources/incorrect.checksum.container</exclude>
<exclude>src/test/resources/incorrect.container</exclude>
<exclude>src/test/resources/test.db.ini</exclude>
+ <exclude>src/main/proto/proto.lock</exclude>
</excludes>
</configuration>
</plugin>
diff --git a/hadoop-hdds/server-scm/pom.xml b/hadoop-hdds/server-scm/pom.xml
index c2b2478..dcbc42a 100644
--- a/hadoop-hdds/server-scm/pom.xml
+++ b/hadoop-hdds/server-scm/pom.xml
@@ -31,6 +31,11 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-test-utils</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-common</artifactId>
</dependency>
@@ -43,7 +48,10 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-client</artifactId>
</dependency>
-
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-hadoop-dependency-server</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-server-framework</artifactId>
@@ -85,10 +93,7 @@
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
- <version>1.3</version>
- <scope>test</scope>
</dependency>
-
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
@@ -107,32 +112,21 @@
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
+ <version>${mockito1-hadoop.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
- <version>1.3</version>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15on</artifactId>
</dependency>
-
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
+ <artifactId>hadoop-hdds-hadoop-dependency-test</artifactId>
<scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
</dependency>
</dependencies>
<build>
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java
index 25457f7..dfacae0 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java
@@ -17,21 +17,24 @@
package org.apache.hadoop.hdds.scm;
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.stream.Collectors;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementStatusDefault;
+import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.net.NetworkTopology;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* This policy implements a set of invariants which are common
* for all basic placement policies, acts as the repository of helper
@@ -43,15 +46,27 @@
LoggerFactory.getLogger(SCMCommonPlacementPolicy.class);
private final NodeManager nodeManager;
private final Random rand;
- private final Configuration conf;
+ private final ConfigurationSource conf;
+
+ /**
+ * Return for replication factor 1 containers where the placement policy
+ * is always met, or not met (zero replicas available) rather than creating a
+ * new object each time. There are only used when there is no network topology
+ * or the replication factor is 1 or the required racks is 1.
+ */
+ private ContainerPlacementStatus validPlacement
+ = new ContainerPlacementStatusDefault(1, 1, 1);
+ private ContainerPlacementStatus invalidPlacement
+ = new ContainerPlacementStatusDefault(0, 1, 1);
/**
* Constructor.
*
* @param nodeManager NodeManager
- * @param conf Configuration class.
+ * @param conf Configuration class.
*/
- public SCMCommonPlacementPolicy(NodeManager nodeManager, Configuration conf) {
+ public SCMCommonPlacementPolicy(NodeManager nodeManager,
+ ConfigurationSource conf) {
this.nodeManager = nodeManager;
this.rand = new Random();
this.conf = conf;
@@ -80,7 +95,7 @@
*
* @return Configuration
*/
- public Configuration getConf() {
+ public ConfigurationSource getConf() {
return conf;
}
@@ -95,11 +110,10 @@
* 3. if a set of containers are requested, we either meet the required
* number of nodes or we fail that request.
*
- *
* @param excludedNodes - datanodes with existing replicas
- * @param favoredNodes - list of nodes preferred.
+ * @param favoredNodes - list of nodes preferred.
* @param nodesRequired - number of datanodes required.
- * @param sizeRequired - size required for the container or block.
+ * @param sizeRequired - size required for the container or block.
* @return list of datanodes chosen.
* @throws SCMException SCM exception.
*/
@@ -162,7 +176,7 @@
* expected number of nodes.
*
* @param nodesRequired - Nodes Required
- * @param healthyNodes - List of Nodes in the result set.
+ * @param healthyNodes - List of Nodes in the result set.
* @return List of Datanodes that can be used for placement.
* @throws SCMException SCMException
*/
@@ -197,4 +211,59 @@
*/
public abstract DatanodeDetails chooseNode(
List<DatanodeDetails> healthyNodes);
+
+ /**
+ * Default implementation to return the number of racks containers should span
+ * to meet the placement policy. For simple policies that are not rack aware
+ * we return 1, from this default implementation.
+ * should have
+ * @return The number of racks containers should span to meet the policy
+ */
+ protected int getRequiredRackCount() {
+ return 1;
+ }
+
+ /**
+ * This default implementation handles rack aware policies and non rack
+ * aware policies. If a future placement policy needs to check more than racks
+ * to validate the policy (eg node groups, HDFS like upgrade domain) this
+ * method should be overridden in the sub class.
+ * This method requires that subclasses which implement rack aware policies
+ * override the default method getRequiredRackCount and getNetworkTopology.
+ * @param dns List of datanodes holding a replica of the container
+ * @param replicas The expected number of replicas
+ * @return ContainerPlacementStatus indicating if the placement policy is
+ * met or not. Not this only considers the rack count and not the
+ * number of replicas.
+ */
+ @Override
+ public ContainerPlacementStatus validateContainerPlacement(
+ List<DatanodeDetails> dns, int replicas) {
+ NetworkTopology topology = nodeManager.getClusterNetworkTopologyMap();
+ int requiredRacks = getRequiredRackCount();
+ if (topology == null || replicas == 1 || requiredRacks == 1) {
+ if (dns.size() > 0) {
+ // placement is always satisfied if there is at least one DN.
+ return validPlacement;
+ } else {
+ return invalidPlacement;
+ }
+ }
+ // We have a network topology so calculate if it is satisfied or not.
+ int numRacks = 1;
+ final int maxLevel = topology.getMaxLevel();
+ // The leaf nodes are all at max level, so the number of nodes at
+ // leafLevel - 1 is the rack count
+ numRacks = topology.getNumOfNodes(maxLevel - 1);
+ final long currentRackCount = dns.stream()
+ .map(d -> topology.getAncestor(d, 1))
+ .distinct()
+ .count();
+
+ if (replicas < requiredRacks) {
+ requiredRacks = replicas;
+ }
+ return new ContainerPlacementStatusDefault(
+ (int)currentRackCount, requiredRacks, numRacks);
+ }
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManager.java
index df26278..b63ee6c 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManager.java
@@ -17,22 +17,24 @@
package org.apache.hadoop.hdds.scm.block;
-import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.client.BlockID;
-import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
-import org.apache.hadoop.hdds.scm.safemode.SafeModeNotification;
-
import java.io.Closeable;
import java.io.IOException;
import java.util.List;
+import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
+import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
+import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager.SafeModeStatus;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+
/**
*
* Block APIs.
* Container is transparent to these APIs.
*/
-public interface BlockManager extends Closeable, SafeModeNotification {
+public interface BlockManager extends Closeable,
+ EventHandler<SafeModeStatus> {
/**
* Allocates a new block for a given size.
* @param size - Block Size
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
index 0f8061c..951dcc0 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
@@ -16,6 +16,7 @@
*/
package org.apache.hadoop.hdds.scm.block;
+import javax.management.ObjectName;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
@@ -23,19 +24,16 @@
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
-import javax.management.ObjectName;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.client.ContainerBlockID;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmOps;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.ScmUtils;
-import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager;
-import org.apache.hadoop.hdds.scm.safemode.SafeModePrecheck;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.ContainerManager;
import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
@@ -44,24 +42,22 @@
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException;
+import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager.SafeModeStatus;
+import org.apache.hadoop.hdds.scm.safemode.SafeModePrecheck;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.hdds.utils.UniqueId;
import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.hdds.utils.UniqueId;
+
+import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes.INVALID_BLOCK_SIZE;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
- .INVALID_BLOCK_SIZE;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_BLOCK_DELETING_SERVICE_TIMEOUT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT;
-
/** Block Manager manages the block access for SCM. */
public class BlockManagerImpl implements BlockManager, BlockmanagerMXBean {
@@ -89,7 +85,7 @@
* @param scm
* @throws IOException
*/
- public BlockManagerImpl(final Configuration conf,
+ public BlockManagerImpl(final ConfigurationSource conf,
final StorageContainerManager scm) {
Objects.requireNonNull(scm, "SCM cannot be null");
this.pipelineManager = scm.getPipelineManager();
@@ -358,11 +354,6 @@
return this.blockDeletingService;
}
- @Override
- public void handleSafeModeTransition(
- SCMSafeModeManager.SafeModeStatus status) {
- this.safeModePrecheck.setInSafeMode(status.getSafeModeStatus());
- }
/**
* Returns status of scm safe mode determined by SAFE_MODE_STATUS event.
* */
@@ -377,6 +368,12 @@
return LOG;
}
+ @Override
+ public void onMessage(SafeModeStatus status,
+ EventPublisher publisher) {
+ this.safeModePrecheck.setInSafeMode(status.isInSafeMode());
+ }
+
/**
* This class uses system current time milliseconds to generate unique id.
*/
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
index 7c920ba..08639ba 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
@@ -17,9 +17,9 @@
*/
package org.apache.hadoop.hdds.scm.block;
-import com.google.common.collect.Lists;
import java.io.IOException;
import java.util.HashMap;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -29,17 +29,13 @@
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
-import org.apache.hadoop.conf.Configuration;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto
- .DeleteBlockTransactionResult;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
-import org.apache.hadoop.hdds.scm.command
- .CommandStatusReportHandler.DeleteBlockStatus;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto.DeleteBlockTransactionResult;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
+import org.apache.hadoop.hdds.scm.command.CommandStatusReportHandler.DeleteBlockStatus;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.ContainerManager;
@@ -50,16 +46,14 @@
import org.apache.hadoop.hdds.utils.db.BatchOperation;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.TableIterator;
-import org.eclipse.jetty.util.ConcurrentHashSet;
+
+import com.google.common.collect.Lists;
+import static java.lang.Math.min;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_DELETION_MAX_RETRY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_DELETION_MAX_RETRY_DEFAULT;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static java.lang.Math.min;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_BLOCK_DELETION_MAX_RETRY;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_BLOCK_DELETION_MAX_RETRY_DEFAULT;
-
/**
* A implement class of {@link DeletedBlockLog}, and it uses
* K/V db to maintain block deletion transactions between scm and datanode.
@@ -83,7 +77,7 @@
// Maps txId to set of DNs which are successful in committing the transaction
private Map<Long, Set<UUID>> transactionToDNsCommitMap;
- public DeletedBlockLogImpl(Configuration conf,
+ public DeletedBlockLogImpl(ConfigurationSource conf,
ContainerManager containerManager,
SCMMetadataStore scmMetadataStore) {
maxRetry = conf.getInt(OZONE_SCM_BLOCK_DELETION_MAX_RETRY,
@@ -344,7 +338,7 @@
deleteTransactionMap.put(block.getContainerID(),
block.getTxID());
transactionToDNsCommitMap
- .putIfAbsent(block.getTxID(), new ConcurrentHashSet<>());
+ .putIfAbsent(block.getTxID(), new LinkedHashSet<>());
}
}
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java
index 74db22d..0980369 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java
@@ -16,37 +16,34 @@
*/
package org.apache.hadoop.hdds.scm.block;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.scm.container.ContainerManager;
-import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
-import org.apache.hadoop.hdds.server.events.EventPublisher;
-import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
-import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.hdds.utils.BackgroundService;
-import org.apache.hadoop.hdds.utils.BackgroundTask;
-import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
-import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL_DEFAULT;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
+import org.apache.hadoop.hdds.scm.container.ContainerManager;
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
+import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
+import org.apache.hadoop.util.Time;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL_DEFAULT;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A background service running in SCM to delete blocks. This service scans
@@ -83,7 +80,7 @@
public SCMBlockDeletingService(DeletedBlockLog deletedBlockLog,
ContainerManager containerManager, NodeManager nodeManager,
EventPublisher eventPublisher, long interval, long serviceTimeout,
- Configuration conf) {
+ ConfigurationSource conf) {
super("SCMBlockDeletingService", interval, TimeUnit.MILLISECONDS,
BLOCK_DELETING_SERVICE_CORE_POOL_SIZE, serviceTimeout);
this.deletedBlockLog = deletedBlockLog;
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java
index 973026d..43c1ced 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java
@@ -16,15 +16,15 @@
*/
package org.apache.hadoop.hdds.scm.container;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
-
import java.io.Closeable;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+
// TODO: Write extensive java doc.
// This is the main interface of ContainerManager.
/**
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
index 2227df6..1b0b81f 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
@@ -18,11 +18,13 @@
package org.apache.hadoop.hdds.scm.container;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.scm.ScmConfig;
import org.apache.hadoop.hdds.scm.block.PendingDeleteStatusList;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.node.NodeManager;
@@ -31,6 +33,8 @@
.ContainerReportFromDatanode;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
+import org.apache.hadoop.ozone.protocol.commands.DeleteContainerCommand;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,6 +55,14 @@
private final NodeManager nodeManager;
private final ContainerManager containerManager;
+ private final String unknownContainerHandleAction;
+
+ /**
+ * The action taken by ContainerReportHandler to handle
+ * unknown containers.
+ */
+ static final String UNKNOWN_CONTAINER_ACTION_WARN = "WARN";
+ static final String UNKNOWN_CONTAINER_ACTION_DELETE = "DELETE";
/**
* Constructs ContainerReportHandler instance with the
@@ -58,12 +70,26 @@
*
* @param nodeManager NodeManager instance
* @param containerManager ContainerManager instance
+ * @param conf OzoneConfiguration instance
*/
public ContainerReportHandler(final NodeManager nodeManager,
- final ContainerManager containerManager) {
+ final ContainerManager containerManager,
+ OzoneConfiguration conf) {
super(containerManager, LOG);
this.nodeManager = nodeManager;
this.containerManager = containerManager;
+
+ if (conf != null) {
+ ScmConfig scmConfig = conf.getObject(ScmConfig.class);
+ unknownContainerHandleAction = scmConfig.getUnknownContainerAction();
+ } else {
+ unknownContainerHandleAction = UNKNOWN_CONTAINER_ACTION_WARN;
+ }
+ }
+
+ public ContainerReportHandler(final NodeManager nodeManager,
+ final ContainerManager containerManager) {
+ this(nodeManager, containerManager, null);
}
/**
@@ -94,7 +120,7 @@
final Set<ContainerID> missingReplicas = new HashSet<>(containersInSCM);
missingReplicas.removeAll(containersInDn);
- processContainerReplicas(datanodeDetails, replicas);
+ processContainerReplicas(datanodeDetails, replicas, publisher);
processMissingReplicas(datanodeDetails, missingReplicas);
updateDeleteTransaction(datanodeDetails, replicas, publisher);
@@ -114,20 +140,37 @@
}
/**
- * Processes the ContainerReport.
+ * Processes the ContainerReport, unknown container reported
+ * that will be deleted by SCM.
*
* @param datanodeDetails Datanode from which this report was received
* @param replicas list of ContainerReplicaProto
+ * @param publisher EventPublisher reference
*/
private void processContainerReplicas(final DatanodeDetails datanodeDetails,
- final List<ContainerReplicaProto> replicas) {
+ final List<ContainerReplicaProto> replicas,
+ final EventPublisher publisher) {
for (ContainerReplicaProto replicaProto : replicas) {
try {
processContainerReplica(datanodeDetails, replicaProto);
} catch (ContainerNotFoundException e) {
- LOG.error("Received container report for an unknown container" +
- " {} from datanode {}.", replicaProto.getContainerID(),
- datanodeDetails, e);
+ if(unknownContainerHandleAction.equals(
+ UNKNOWN_CONTAINER_ACTION_WARN)) {
+ LOG.error("Received container report for an unknown container" +
+ " {} from datanode {}.", replicaProto.getContainerID(),
+ datanodeDetails, e);
+ } else if (unknownContainerHandleAction.equals(
+ UNKNOWN_CONTAINER_ACTION_DELETE)) {
+ final ContainerID containerId = ContainerID
+ .valueof(replicaProto.getContainerID());
+ final DeleteContainerCommand deleteCommand =
+ new DeleteContainerCommand(containerId.getId(), true);
+ final CommandForDatanode datanodeCommand = new CommandForDatanode<>(
+ datanodeDetails.getUuid(), deleteCommand);
+ publisher.fireEvent(SCMEvents.DATANODE_COMMAND, datanodeCommand);
+ LOG.info("Sending delete container command for unknown container {}"
+ + " to datanode {}", containerId.getId(), datanodeDetails);
+ }
} catch (IOException e) {
LOG.error("Exception while processing container report for container" +
" {} from datanode {}.", replicaProto.getContainerID(),
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java
index 36259c4..5a22521 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java
@@ -17,9 +17,6 @@
package org.apache.hadoop.hdds.scm.container;
-import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
- .FAILED_TO_CHANGE_CONTAINER_STATE;
-
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
@@ -29,8 +26,8 @@
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.StorageUnit;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
@@ -43,15 +40,15 @@
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
-import org.apache.hadoop.ozone.common.statemachine
- .InvalidStateTransitionException;
+import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException;
import org.apache.hadoop.ozone.common.statemachine.StateMachine;
import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
import com.google.common.util.concurrent.AtomicLongMap;
+import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes.FAILED_TO_CHANGE_CONTAINER_STATE;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A container state manager keeps track of container states and returns
@@ -132,7 +129,7 @@
* TODO : Add Container Tags so we know which containers are owned by SCM.
*/
@SuppressWarnings("unchecked")
- public ContainerStateManager(final Configuration configuration) {
+ public ContainerStateManager(final ConfigurationSource configuration) {
// Initialize the container state machine.
final Set<HddsProtos.LifeCycleState> finalStates = new HashSet();
@@ -308,7 +305,7 @@
.setPipelineID(pipeline.getId())
.setUsedBytes(0)
.setNumberOfKeys(0)
- .setStateEnterTime(Time.monotonicNow())
+ .setStateEnterTime(Time.now())
.setOwner(owner)
.setContainerID(containerID)
.setDeleteTransactionId(0)
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java
index 6972cef..76155d2 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java
@@ -21,6 +21,7 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
@@ -38,12 +39,12 @@
import org.apache.hadoop.hdds.conf.ConfigType;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State;
+import org.apache.hadoop.hdds.scm.ContainerPlacementStatus;
import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager;
-import org.apache.hadoop.hdds.scm.safemode.SafeModeNotification;
+import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager.SafeModeStatus;
+import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsInfo;
@@ -72,7 +73,8 @@
* that the containers are properly replicated. Replication Manager deals only
* with Quasi Closed / Closed container.
*/
-public class ReplicationManager implements MetricsSource, SafeModeNotification {
+public class ReplicationManager
+ implements MetricsSource, EventHandler<SafeModeStatus> {
private static final Logger LOG =
LoggerFactory.getLogger(ReplicationManager.class);
@@ -252,9 +254,14 @@
final LifeCycleState state = container.getState();
/*
- * We don't take any action if the container is in OPEN state.
+ * We don't take any action if the container is in OPEN state and
+ * the container is healthy. If the container is not healthy, i.e.
+ * the replicas are not in OPEN state, send CLOSE_CONTAINER command.
*/
if (state == LifeCycleState.OPEN) {
+ if (!isContainerHealthy(container, replicas)) {
+ eventPublisher.fireEvent(SCMEvents.CLOSE_CONTAINER, id);
+ }
return;
}
@@ -374,7 +381,8 @@
*/
private boolean isContainerHealthy(final ContainerInfo container,
final Set<ContainerReplica> replicas) {
- return container.getReplicationFactor().getNumber() == replicas.size() &&
+ return !isContainerUnderReplicated(container, replicas) &&
+ !isContainerOverReplicated(container, replicas) &&
replicas.stream().allMatch(
r -> compareState(container.getState(), r.getState()));
}
@@ -388,8 +396,11 @@
*/
private boolean isContainerUnderReplicated(final ContainerInfo container,
final Set<ContainerReplica> replicas) {
+ boolean misReplicated = !getPlacementStatus(
+ replicas, container.getReplicationFactor().getNumber())
+ .isPolicySatisfied();
return container.getReplicationFactor().getNumber() >
- getReplicaCount(container.containerID(), replicas);
+ getReplicaCount(container.containerID(), replicas) || misReplicated;
}
/**
@@ -496,6 +507,11 @@
.stream()
.map(action -> action.datanode)
.collect(Collectors.toList());
+ final List<DatanodeDetails> replicationInFlight = inflightReplication
+ .getOrDefault(id, Collections.emptyList())
+ .stream()
+ .map(action -> action.datanode)
+ .collect(Collectors.toList());
final List<DatanodeDetails> source = replicas.stream()
.filter(r ->
r.getState() == State.QUASI_CLOSED ||
@@ -507,25 +523,56 @@
if (source.size() > 0) {
final int replicationFactor = container
.getReplicationFactor().getNumber();
- final int delta = replicationFactor - getReplicaCount(id, replicas);
+ // Want to check if the container is mis-replicated after considering
+ // inflight add and delete.
+ // Create a new list from source (healthy replicas minus pending delete)
+ List<DatanodeDetails> targetReplicas = new ArrayList<>(source);
+ // Then add any pending additions
+ targetReplicas.addAll(replicationInFlight);
+
+ int delta = replicationFactor - getReplicaCount(id, replicas);
+ final ContainerPlacementStatus placementStatus =
+ containerPlacement.validateContainerPlacement(
+ targetReplicas, replicationFactor);
+ final int misRepDelta = placementStatus.misReplicationCount();
+ final int replicasNeeded
+ = delta < misRepDelta ? misRepDelta : delta;
+
final List<DatanodeDetails> excludeList = replicas.stream()
.map(ContainerReplica::getDatanodeDetails)
.collect(Collectors.toList());
- List<InflightAction> actionList = inflightReplication.get(id);
- if (actionList != null) {
- actionList.stream().map(r -> r.datanode)
- .forEach(excludeList::add);
- }
+ excludeList.addAll(replicationInFlight);
final List<DatanodeDetails> selectedDatanodes = containerPlacement
- .chooseDatanodes(excludeList, null, delta,
+ .chooseDatanodes(excludeList, null, replicasNeeded,
container.getUsedBytes());
-
- LOG.info("Container {} is under replicated. Expected replica count" +
- " is {}, but found {}.", id, replicationFactor,
- replicationFactor - delta);
-
- for (DatanodeDetails datanode : selectedDatanodes) {
- sendReplicateCommand(container, datanode, source);
+ if (delta > 0) {
+ LOG.info("Container {} is under replicated. Expected replica count" +
+ " is {}, but found {}.", id, replicationFactor,
+ replicationFactor - delta);
+ }
+ int newMisRepDelta = misRepDelta;
+ if (misRepDelta > 0) {
+ LOG.info("Container: {}. {}",
+ id, placementStatus.misReplicatedReason());
+ // Check if the new target nodes (original plus newly selected nodes)
+ // makes the placement policy valid.
+ targetReplicas.addAll(selectedDatanodes);
+ newMisRepDelta = containerPlacement.validateContainerPlacement(
+ targetReplicas, replicationFactor).misReplicationCount();
+ }
+ if (delta > 0 || newMisRepDelta < misRepDelta) {
+ // Only create new replicas if we are missing a replicas or
+ // the number of pending mis-replication has improved. No point in
+ // creating new replicas for mis-replicated containers unless it
+ // improves things.
+ for (DatanodeDetails datanode : selectedDatanodes) {
+ sendReplicateCommand(container, datanode, source);
+ }
+ } else {
+ LOG.warn("Container {} is mis-replicated, requiring {} additional " +
+ "replicas. After selecting new nodes, mis-replication has not " +
+ "improved. No additional replicas will be scheduled",
+ id, misRepDelta);
}
} else {
LOG.warn("Cannot replicate container {}, no healthy replica found.",
@@ -551,7 +598,7 @@
final ContainerID id = container.containerID();
final int replicationFactor = container.getReplicationFactor().getNumber();
// Dont consider inflight replication while calculating excess here.
- final int excess = replicas.size() - replicationFactor -
+ int excess = replicas.size() - replicationFactor -
inflightDeletion.getOrDefault(id, Collections.emptyList()).size();
if (excess > 0) {
@@ -577,18 +624,72 @@
.filter(r -> !compareState(container.getState(), r.getState()))
.collect(Collectors.toList());
- //Move the unhealthy replicas to the front of eligible replicas to delete
- eligibleReplicas.removeAll(unhealthyReplicas);
- eligibleReplicas.addAll(0, unhealthyReplicas);
-
- for (int i = 0; i < excess; i++) {
- sendDeleteCommand(container,
- eligibleReplicas.get(i).getDatanodeDetails(), true);
+ // If there are unhealthy replicas, then we should remove them even if it
+ // makes the container violate the placement policy, as excess unhealthy
+ // containers are not really useful. It will be corrected later as a
+ // mis-replicated container will be seen as under-replicated.
+ for (ContainerReplica r : unhealthyReplicas) {
+ if (excess > 0) {
+ sendDeleteCommand(container, r.getDatanodeDetails(), true);
+ excess -= 1;
+ }
+ break;
+ }
+ // After removing all unhealthy replicas, if the container is still over
+ // replicated then we need to check if it is already mis-replicated.
+ // If it is, we do no harm by removing excess replicas. However, if it is
+ // not mis-replicated, then we can only remove replicas if they don't
+ // make the container become mis-replicated.
+ if (excess > 0) {
+ eligibleReplicas.removeAll(unhealthyReplicas);
+ Set<ContainerReplica> replicaSet = new HashSet<>(eligibleReplicas);
+ boolean misReplicated =
+ getPlacementStatus(replicaSet, replicationFactor)
+ .isPolicySatisfied();
+ for (ContainerReplica r : eligibleReplicas) {
+ if (excess <= 0) {
+ break;
+ }
+ // First remove the replica we are working on from the set, and then
+ // check if the set is now mis-replicated.
+ replicaSet.remove(r);
+ boolean nowMisRep = getPlacementStatus(replicaSet, replicationFactor)
+ .isPolicySatisfied();
+ if (misReplicated || !nowMisRep) {
+ // Remove the replica if the container was already mis-replicated
+ // OR if losing this replica does not make it become mis-replicated
+ sendDeleteCommand(container, r.getDatanodeDetails(), true);
+ excess -= 1;
+ continue;
+ }
+ // If we decided not to remove this replica, put it back into the set
+ replicaSet.add(r);
+ }
+ if (excess > 0) {
+ LOG.info("The container {} is over replicated with {} excess " +
+ "replica. The excess replicas cannot be removed without " +
+ "violating the placement policy", container, excess);
+ }
}
}
}
/**
+ * Given a set of ContainerReplica, transform it to a list of DatanodeDetails
+ * and then check if the list meets the container placement policy.
+ * @param replicas List of containerReplica
+ * @param replicationFactor Expected Replication Factor of the containe
+ * @return ContainerPlacementStatus indicating if the policy is met or not
+ */
+ private ContainerPlacementStatus getPlacementStatus(
+ Set<ContainerReplica> replicas, int replicationFactor) {
+ List<DatanodeDetails> replicaDns = replicas.stream()
+ .map(c -> c.getDatanodeDetails()).collect(Collectors.toList());
+ return containerPlacement.validateContainerPlacement(
+ replicaDns, replicationFactor);
+ }
+
+ /**
* Handles unstable container.
* A container is inconsistent if any of the replica state doesn't
* match the container state. We have to take appropriate action
@@ -776,9 +877,9 @@
}
@Override
- public void handleSafeModeTransition(
- SCMSafeModeManager.SafeModeStatus status) {
- if (!status.getSafeModeStatus() && !this.isRunning()) {
+ public void onMessage(SafeModeStatus status,
+ EventPublisher publisher) {
+ if (!status.isInSafeMode() && !this.isRunning()) {
this.start();
}
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java
index 3838b9d..9f47608 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java
@@ -16,30 +16,6 @@
*/
package org.apache.hadoop.hdds.scm.container;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.primitives.Longs;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ContainerInfoProto;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.container.metrics.SCMContainerManagerMetrics;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
-import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
-import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException;
-import org.apache.hadoop.hdds.server.ServerUtils;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.hdds.utils.BatchOperation;
-import org.apache.hadoop.hdds.utils.MetadataStore;
-import org.apache.hadoop.hdds.utils.MetadataStoreBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
@@ -53,9 +29,26 @@
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_DEFAULT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_MB;
-import static org.apache.hadoop.ozone.OzoneConsts.SCM_CONTAINER_DB;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.metrics.SCMContainerManagerMetrics;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException;
+import org.apache.hadoop.hdds.utils.db.BatchOperationHandler;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.Table.KeyValue;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* ContainerManager class contains the mapping from a name to a pipeline
@@ -67,38 +60,40 @@
SCMContainerManager.class);
private final Lock lock;
- private final MetadataStore containerStore;
+
private final PipelineManager pipelineManager;
+
private final ContainerStateManager containerStateManager;
+
private final int numContainerPerOwnerInPipeline;
private final SCMContainerManagerMetrics scmContainerManagerMetrics;
+ private Table<ContainerID, ContainerInfo> containerStore;
+
+ private BatchOperationHandler batchHandler;
+
/**
* Constructs a mapping class that creates mapping between container names
* and pipelines.
- *
+ * <p>
* passed to LevelDB and this memory is allocated in Native code space.
* CacheSize is specified
* in MB.
- * @param conf - {@link Configuration}
+ *
+ * @param conf - {@link ConfigurationSource}
* @param pipelineManager - {@link PipelineManager}
* @throws IOException on Failure.
*/
- public SCMContainerManager(final Configuration conf,
+ public SCMContainerManager(
+ final ConfigurationSource conf,
+ Table<ContainerID, ContainerInfo> containerStore,
+ BatchOperationHandler batchHandler,
PipelineManager pipelineManager)
throws IOException {
- final File containerDBPath = getContainerDBPath(conf);
- final int cacheSize = conf.getInt(OZONE_SCM_DB_CACHE_SIZE_MB,
- OZONE_SCM_DB_CACHE_SIZE_DEFAULT);
-
- this.containerStore = MetadataStoreBuilder.newBuilder()
- .setConf(conf)
- .setDbFile(containerDBPath)
- .setCacheSize(cacheSize * OzoneConsts.MB)
- .build();
-
+ this.batchHandler = batchHandler;
+ this.containerStore = containerStore;
this.lock = new ReentrantLock();
this.pipelineManager = pipelineManager;
this.containerStateManager = new ContainerStateManager(conf);
@@ -112,11 +107,12 @@
}
private void loadExistingContainers() throws IOException {
- List<Map.Entry<byte[], byte[]>> range = containerStore
- .getSequentialRangeKVs(null, Integer.MAX_VALUE, null);
- for (Map.Entry<byte[], byte[]> entry : range) {
- ContainerInfo container = ContainerInfo.fromProtobuf(
- ContainerInfoProto.PARSER.parseFrom(entry.getValue()));
+
+ TableIterator<ContainerID, ? extends KeyValue<ContainerID, ContainerInfo>>
+ iterator = containerStore.iterator();
+
+ while (iterator.hasNext()) {
+ ContainerInfo container = iterator.next().getValue();
Preconditions.checkNotNull(container);
containerStateManager.loadContainer(container);
try {
@@ -304,10 +300,8 @@
lock.lock();
try {
containerStateManager.removeContainer(containerID);
- final byte[] dbKey = Longs.toByteArray(containerID.getId());
- final byte[] containerBytes = containerStore.get(dbKey);
- if (containerBytes != null) {
- containerStore.delete(dbKey);
+ if (containerStore.get(containerID) != null) {
+ containerStore.delete(containerID);
} else {
// Where did the container go? o_O
LOG.warn("Unable to remove the container {} from container store," +
@@ -358,8 +352,7 @@
containerID);
}
}
- final byte[] dbKey = Longs.toByteArray(containerID.getId());
- containerStore.put(dbKey, container.getProtobuf().toByteArray());
+ containerStore.put(containerID, container);
return newState;
} catch (ContainerNotFoundException cnfe) {
throw new SCMException(
@@ -372,38 +365,40 @@
}
}
- /**
- * Update deleteTransactionId according to deleteTransactionMap.
- *
- * @param deleteTransactionMap Maps the containerId to latest delete
- * transaction id for the container.
- * @throws IOException
- */
+ /**
+ * Update deleteTransactionId according to deleteTransactionMap.
+ *
+ * @param deleteTransactionMap Maps the containerId to latest delete
+ * transaction id for the container.
+ * @throws IOException
+ */
public void updateDeleteTransactionId(Map<Long, Long> deleteTransactionMap)
throws IOException {
+
if (deleteTransactionMap == null) {
return;
}
-
+ org.apache.hadoop.hdds.utils.db.BatchOperation batchOperation =
+ batchHandler.initBatchOperation();
lock.lock();
try {
- BatchOperation batch = new BatchOperation();
for (Map.Entry<Long, Long> entry : deleteTransactionMap.entrySet()) {
long containerID = entry.getKey();
- byte[] dbKey = Longs.toByteArray(containerID);
- byte[] containerBytes = containerStore.get(dbKey);
- if (containerBytes == null) {
+
+ ContainerID containerIdObject = new ContainerID(containerID);
+ ContainerInfo containerInfo =
+ containerStore.get(containerIdObject);
+ if (containerInfo == null) {
throw new SCMException(
"Failed to increment number of deleted blocks for container "
+ containerID + ", reason : " + "container doesn't exist.",
SCMException.ResultCodes.FAILED_TO_FIND_CONTAINER);
}
- ContainerInfo containerInfo = ContainerInfo.fromProtobuf(
- HddsProtos.ContainerInfoProto.parseFrom(containerBytes));
containerInfo.updateDeleteTransactionId(entry.getValue());
- batch.put(dbKey, containerInfo.getProtobuf().toByteArray());
+ containerStore
+ .putWithBatch(batchOperation, containerIdObject, containerInfo);
}
- containerStore.writeBatch(batch);
+ batchHandler.commitBatchOperation(batchOperation);
containerStateManager
.updateDeleteTransactionId(deleteTransactionMap);
} finally {
@@ -477,10 +472,8 @@
protected void addContainerToDB(ContainerInfo containerInfo)
throws IOException {
try {
- final byte[] containerIDBytes = Longs.toByteArray(
- containerInfo.getContainerID());
- containerStore.put(containerIDBytes,
- containerInfo.getProtobuf().toByteArray());
+ containerStore
+ .put(new ContainerID(containerInfo.getContainerID()), containerInfo);
// Incrementing here, as allocateBlock to create a container calls
// getMatchingContainer() and finally calls this API to add newly
// created container to DB.
@@ -586,9 +579,6 @@
if (containerStateManager != null) {
containerStateManager.close();
}
- if (containerStore != null) {
- containerStore.close();
- }
if (scmContainerManagerMetrics != null) {
this.scmContainerManagerMetrics.unRegister();
@@ -612,11 +602,6 @@
}
}
- protected File getContainerDBPath(Configuration conf) {
- File metaDir = ServerUtils.getScmDbDir(conf);
- return new File(metaDir, SCM_CONTAINER_DB);
- }
-
protected PipelineManager getPipelineManager() {
return pipelineManager;
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java
index a3024df..6833765 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java
@@ -16,17 +16,18 @@
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.container.placement.algorithms;
-import org.apache.hadoop.conf.Configuration;
+import java.lang.reflect.Constructor;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.net.NetworkTopology;
import org.apache.hadoop.hdds.scm.node.NodeManager;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.lang.reflect.Constructor;
-
/**
* A factory to create container placement instance based on configuration
* property ozone.scm.container.placement.classname.
@@ -44,7 +45,7 @@
public static PlacementPolicy getPolicy(
- Configuration conf, final NodeManager nodeManager,
+ ConfigurationSource conf, final NodeManager nodeManager,
NetworkTopology clusterMap, final boolean fallback,
SCMContainerPlacementMetrics metrics) throws SCMException{
final Class<? extends PlacementPolicy> placementClass = conf
@@ -54,7 +55,7 @@
Constructor<? extends PlacementPolicy> constructor;
try {
constructor = placementClass.getDeclaredConstructor(NodeManager.class,
- Configuration.class, NetworkTopology.class, boolean.class,
+ ConfigurationSource.class, NetworkTopology.class, boolean.class,
SCMContainerPlacementMetrics.class);
LOG.info("Create container placement policy of type {}",
placementClass.getCanonicalName());
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementStatusDefault.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementStatusDefault.java
new file mode 100644
index 0000000..c9528ec
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementStatusDefault.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.scm.container.placement.algorithms;
+
+import org.apache.hadoop.hdds.scm.ContainerPlacementStatus;
+
+/**
+ * Simple Status object to check if a container is replicated across enough
+ * racks.
+ */
+public class ContainerPlacementStatusDefault
+ implements ContainerPlacementStatus {
+
+ private final int requiredRacks;
+ private final int currentRacks;
+ private final int totalRacks;
+
+ public ContainerPlacementStatusDefault(int currentRacks, int requiredRacks,
+ int totalRacks) {
+ this.requiredRacks = requiredRacks;
+ this.currentRacks = currentRacks;
+ this.totalRacks = totalRacks;
+ }
+
+ @Override
+ public boolean isPolicySatisfied() {
+ return currentRacks >= totalRacks || currentRacks >= requiredRacks;
+ }
+
+ @Override
+ public String misReplicatedReason() {
+ if (isPolicySatisfied()) {
+ return null;
+ }
+ return "The container is mis-replicated as it is on " + currentRacks +
+ " racks but should be on " + requiredRacks + " racks.";
+ }
+
+ @Override
+ public int misReplicationCount() {
+ if (isPolicySatisfied()) {
+ return 0;
+ }
+ return requiredRacks - currentRacks;
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java
index 1909344..7d2db05 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java
@@ -19,7 +19,7 @@
import java.util.List;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.SCMCommonPlacementPolicy;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
@@ -80,7 +80,7 @@
* @param conf Configuration
*/
public SCMContainerPlacementCapacity(final NodeManager nodeManager,
- final Configuration conf, final NetworkTopology networkTopology,
+ final ConfigurationSource conf, final NetworkTopology networkTopology,
final boolean fallback, final SCMContainerPlacementMetrics metrics) {
super(nodeManager, conf);
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackAware.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackAware.java
index 8933fe9..113b518 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackAware.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackAware.java
@@ -19,7 +19,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.SCMCommonPlacementPolicy;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
@@ -56,6 +56,8 @@
private static final int RACK_LEVEL = 1;
private static final int MAX_RETRY= 3;
private final SCMContainerPlacementMetrics metrics;
+ // Used to check the placement policy is validated in the parent class
+ private static final int REQUIRED_RACKS = 2;
/**
* Constructs a Container Placement with rack awareness.
@@ -68,7 +70,7 @@
* for closed container placement.
*/
public SCMContainerPlacementRackAware(final NodeManager nodeManager,
- final Configuration conf, final NetworkTopology networkTopology,
+ final ConfigurationSource conf, final NetworkTopology networkTopology,
final boolean fallback, final SCMContainerPlacementMetrics metrics) {
super(nodeManager, conf);
this.networkTopology = networkTopology;
@@ -345,4 +347,9 @@
}
}
}
+
+ @Override
+ protected int getRequiredRackCount() {
+ return REQUIRED_RACKS;
+ }
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java
index ce5d10d..4927517 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.hdds.scm.container.placement.algorithms;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.SCMCommonPlacementPolicy;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
@@ -52,7 +52,7 @@
* @param conf Config
*/
public SCMContainerPlacementRandom(final NodeManager nodeManager,
- final Configuration conf, final NetworkTopology networkTopology,
+ final ConfigurationSource conf, final NetworkTopology networkTopology,
final boolean fallback, final SCMContainerPlacementMetrics metrics) {
super(nodeManager, conf);
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/ContainerStat.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/ContainerStat.java
index b8e8998..9255303 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/ContainerStat.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/ContainerStat.java
@@ -19,7 +19,7 @@
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
+import org.apache.hadoop.hdds.server.JsonUtils;
import java.io.IOException;
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
index 6de05fd..8b40571 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
@@ -21,26 +21,18 @@
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.block.PendingDeleteStatusList;
-import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
-import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager.SafeModeStatus;
import org.apache.hadoop.hdds.scm.command.CommandStatusReportHandler;
import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
- .IncrementalContainerReportFromDatanode;
-import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
- .PipelineReportFromDatanode;
-import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
- .PipelineActionsFromDatanode;
-import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
- .ContainerActionsFromDatanode;
-import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
- .CommandStatusReportFromDatanode;
-import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
- .ContainerReportFromDatanode;
-import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
- .NodeReportFromDatanode;
-import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer
- .NodeRegistrationContainerReport;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager.SafeModeStatus;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.CommandStatusReportFromDatanode;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.ContainerActionsFromDatanode;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.ContainerReportFromDatanode;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.IncrementalContainerReportFromDatanode;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.NodeReportFromDatanode;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineActionsFromDatanode;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineReportFromDatanode;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer.NodeRegistrationContainerReport;
import org.apache.hadoop.hdds.server.events.Event;
import org.apache.hadoop.hdds.server.events.TypedEvent;
import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
@@ -194,7 +186,10 @@
new TypedEvent<>(PendingDeleteStatusList.class, "Pending_Delete_Status");
public static final TypedEvent<SafeModeStatus> SAFE_MODE_STATUS =
- new TypedEvent<>(SafeModeStatus.class);
+ new TypedEvent<>(SafeModeStatus.class, "Safe mode status");
+
+ public static final TypedEvent<SafeModeStatus> DELAYED_SAFE_MODE_STATUS =
+ new TypedEvent<>(SafeModeStatus.class, "Delayed safe mode status");
/**
* Private Ctor. Never Constructed.
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/ContainerIDCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/ContainerIDCodec.java
new file mode 100644
index 0000000..87c9e91
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/ContainerIDCodec.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.scm.metadata;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.utils.db.Codec;
+import org.apache.hadoop.hdds.utils.db.LongCodec;
+
+/**
+ * Codec to serialize / deserialize ContainerID.
+ */
+public class ContainerIDCodec implements Codec<ContainerID> {
+
+ private Codec<Long> longCodec = new LongCodec();
+
+ @Override
+ public byte[] toPersistedFormat(ContainerID container) throws IOException {
+ return longCodec.toPersistedFormat(container.getId());
+ }
+
+ @Override
+ public ContainerID fromPersistedFormat(byte[] rawData) throws IOException {
+ return new ContainerID(longCodec.fromPersistedFormat(rawData));
+ }
+
+ @Override
+ public ContainerID copyObject(ContainerID object) {
+ return new ContainerID(object.getId());
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/ContainerInfoCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/ContainerInfoCodec.java
new file mode 100644
index 0000000..6b26215
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/ContainerInfoCodec.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.scm.metadata;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ContainerInfoProto;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.utils.db.Codec;
+
+/**
+ * Codec to serialize / deserialize ContainerInfo.
+ */
+public class ContainerInfoCodec implements Codec<ContainerInfo> {
+
+ @Override
+ public byte[] toPersistedFormat(ContainerInfo container) throws IOException {
+ return container.getProtobuf().toByteArray();
+ }
+
+ @Override
+ public ContainerInfo fromPersistedFormat(byte[] rawData) throws IOException {
+ return ContainerInfo.fromProtobuf(
+ ContainerInfoProto.PARSER.parseFrom(rawData));
+ }
+
+ @Override
+ public ContainerInfo copyObject(ContainerInfo object) {
+ throw new UnsupportedOperationException();
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/PipelineCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/PipelineCodec.java
new file mode 100644
index 0000000..25a1e44
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/PipelineCodec.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.scm.metadata;
+
+import java.io.IOException;
+import java.time.Instant;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.utils.db.Codec;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Codec to serialize / deserialize Pipeline.
+ */
+public class PipelineCodec implements Codec<Pipeline> {
+
+ @Override
+ public byte[] toPersistedFormat(Pipeline object) throws IOException {
+ return object.getProtobufMessage().toByteArray();
+ }
+
+ @Override
+ public Pipeline fromPersistedFormat(byte[] rawData) throws IOException {
+ HddsProtos.Pipeline.Builder pipelineBuilder = HddsProtos.Pipeline
+ .newBuilder(HddsProtos.Pipeline.PARSER.parseFrom(rawData));
+ Pipeline pipeline = Pipeline.getFromProtobuf(pipelineBuilder.setState(
+ HddsProtos.PipelineState.PIPELINE_ALLOCATED).build());
+ // When SCM is restarted, set Creation time with current time.
+ pipeline.setCreationTimestamp(Instant.now());
+ Preconditions.checkNotNull(pipeline);
+ return pipeline;
+ }
+
+ @Override
+ public Pipeline copyObject(Pipeline object) {
+ throw new UnsupportedOperationException();
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/PipelineIDCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/PipelineIDCodec.java
new file mode 100644
index 0000000..d661e34
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/PipelineIDCodec.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.scm.metadata;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.hdds.utils.db.Codec;
+
+/**
+ * Codec to serialize / deserialize PipelineID.
+ */
+public class PipelineIDCodec implements Codec<PipelineID> {
+
+ @Override
+ public byte[] toPersistedFormat(PipelineID object) throws IOException {
+ return object.getProtobuf().toByteArray();
+ }
+
+ @Override
+ public PipelineID fromPersistedFormat(byte[] rawData) throws IOException {
+ return null;
+ }
+
+ @Override
+ public PipelineID copyObject(PipelineID object) {
+ throw new UnsupportedOperationException();
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java
new file mode 100644
index 0000000..fcddcdd
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.metadata;
+
+import java.math.BigInteger;
+import java.security.cert.X509Certificate;
+
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition;
+import org.apache.hadoop.hdds.utils.db.DBDefinition;
+import org.apache.hadoop.hdds.utils.db.LongCodec;
+
+/**
+ * Class defines the structure and types of the scm.db.
+ */
+public class SCMDBDefinition implements DBDefinition {
+
+ public static final DBColumnFamilyDefinition<Long, DeletedBlocksTransaction>
+ DELETED_BLOCKS =
+ new DBColumnFamilyDefinition<>(
+ "deletedBlocks",
+ Long.class,
+ new LongCodec(),
+ DeletedBlocksTransaction.class,
+ new DeletedBlocksTransactionCodec());
+
+ public static final DBColumnFamilyDefinition<BigInteger, X509Certificate>
+ VALID_CERTS =
+ new DBColumnFamilyDefinition<>(
+ "validCerts",
+ BigInteger.class,
+ new BigIntegerCodec(),
+ X509Certificate.class,
+ new X509CertificateCodec());
+
+ public static final DBColumnFamilyDefinition<BigInteger, X509Certificate>
+ REVOKED_CERTS =
+ new DBColumnFamilyDefinition<>(
+ "revokedCerts",
+ BigInteger.class,
+ new BigIntegerCodec(),
+ X509Certificate.class,
+ new X509CertificateCodec());
+
+ public static final DBColumnFamilyDefinition<PipelineID, Pipeline>
+ PIPELINES =
+ new DBColumnFamilyDefinition<>(
+ "pipelines",
+ PipelineID.class,
+ new PipelineIDCodec(),
+ Pipeline.class,
+ new PipelineCodec());
+
+ public static final DBColumnFamilyDefinition<ContainerID, ContainerInfo>
+ CONTAINERS =
+ new DBColumnFamilyDefinition<ContainerID, ContainerInfo>(
+ "containers",
+ ContainerID.class,
+ new ContainerIDCodec(),
+ ContainerInfo.class,
+ new ContainerInfoCodec());
+
+ @Override
+ public String getName() {
+ return "scm.db";
+ }
+
+ @Override
+ public String getLocationConfigKey() {
+ return ScmConfigKeys.OZONE_SCM_DB_DIRS;
+ }
+
+ @Override
+ public DBColumnFamilyDefinition[] getColumnFamilies() {
+ return new DBColumnFamilyDefinition[] {DELETED_BLOCKS, VALID_CERTS,
+ REVOKED_CERTS, PIPELINES, CONTAINERS};
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStore.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStore.java
index 1150316..0452c05 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStore.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStore.java
@@ -17,18 +17,24 @@
*/
package org.apache.hadoop.hdds.scm.metadata;
+import java.io.IOException;
import java.math.BigInteger;
import java.security.cert.X509Certificate;
+
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import java.io.IOException;
-import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore;
+import org.apache.hadoop.hdds.utils.db.BatchOperationHandler;
import org.apache.hadoop.hdds.utils.db.DBStore;
import org.apache.hadoop.hdds.utils.db.Table;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
import org.apache.hadoop.hdds.utils.db.TableIterator;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* Generic interface for data stores for SCM.
* This is similar to the OMMetadataStore class,
@@ -99,5 +105,18 @@
*/
TableIterator getAllCerts(CertificateStore.CertType certType);
+ /**
+ * A Table that maintains all the pipeline information.
+ */
+ Table<PipelineID, Pipeline> getPipelineTable();
+ /**
+ * Helper to create and write batch transactions.
+ */
+ BatchOperationHandler getBatchHandler();
+
+ /**
+ * Table that maintains all the container information.
+ */
+ Table<ContainerID, ContainerInfo> getContainerTable();
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreRDBImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreRDBImpl.java
index 72818a3..3823fd8 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreRDBImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreRDBImpl.java
@@ -17,53 +17,48 @@
*/
package org.apache.hadoop.hdds.scm.metadata;
-import java.io.File;
+import java.io.IOException;
import java.math.BigInteger;
-import java.nio.file.Paths;
import java.security.cert.X509Certificate;
import java.util.concurrent.atomic.AtomicLong;
+
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import java.io.IOException;
-import org.apache.hadoop.hdds.security.x509.certificate.authority
- .CertificateStore;
-import org.apache.hadoop.hdds.server.ServerUtils;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore;
+import org.apache.hadoop.hdds.utils.db.BatchOperationHandler;
import org.apache.hadoop.hdds.utils.db.DBStore;
import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
import org.apache.hadoop.hdds.utils.db.Table;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
+import org.apache.hadoop.hdds.utils.db.Table.KeyValue;
import org.apache.hadoop.hdds.utils.db.TableIterator;
+
+import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.CONTAINERS;
+import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.DELETED_BLOCKS;
+import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.PIPELINES;
+import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.REVOKED_CERTS;
+import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.VALID_CERTS;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.hadoop.ozone.OzoneConsts.SCM_DB_NAME;
-
/**
* A RocksDB based implementation of SCM Metadata Store.
- * <p>
- * <p>
- * +---------------+------------------+-------------------------+
- * | Column Family | Key | Value |
- * +---------------+------------------+-------------------------+
- * | DeletedBlocks | TXID(Long) | DeletedBlockTransaction |
- * +---------------+------------------+-------------------------+
- * | ValidCerts | Serial (BigInt) | X509Certificate |
- * +---------------+------------------+-------------------------+
- * |RevokedCerts | Serial (BigInt) | X509Certificate |
- * +---------------+------------------+-------------------------+
+ *
*/
public class SCMMetadataStoreRDBImpl implements SCMMetadataStore {
- private static final String DELETED_BLOCKS_TABLE = "deletedBlocks";
- private Table deletedBlocksTable;
+ private Table<Long, DeletedBlocksTransaction> deletedBlocksTable;
- private static final String VALID_CERTS_TABLE = "validCerts";
- private Table validCertsTable;
+ private Table<BigInteger, X509Certificate> validCertsTable;
- private static final String REVOKED_CERTS_TABLE = "revokedCerts";
- private Table revokedCertsTable;
+ private Table<BigInteger, X509Certificate> revokedCertsTable;
+ private Table<ContainerID, ContainerInfo> containerTable;
+ private Table<PipelineID, Pipeline> pipelineTable;
private static final Logger LOG =
LoggerFactory.getLogger(SCMMetadataStoreRDBImpl.class);
@@ -88,31 +83,26 @@
public void start(OzoneConfiguration config)
throws IOException {
if (this.store == null) {
- File metaDir = ServerUtils.getScmDbDir(configuration);
- this.store = DBStoreBuilder.newBuilder(configuration)
- .setName(SCM_DB_NAME)
- .setPath(Paths.get(metaDir.getPath()))
- .addTable(DELETED_BLOCKS_TABLE)
- .addTable(VALID_CERTS_TABLE)
- .addTable(REVOKED_CERTS_TABLE)
- .addCodec(DeletedBlocksTransaction.class,
- new DeletedBlocksTransactionCodec())
- .addCodec(BigInteger.class, new BigIntegerCodec())
- .addCodec(X509Certificate.class, new X509CertificateCodec())
- .build();
+ this.store = DBStoreBuilder.createDBStore(config, new SCMDBDefinition());
- deletedBlocksTable = this.store.getTable(DELETED_BLOCKS_TABLE,
- Long.class, DeletedBlocksTransaction.class);
- checkTableStatus(deletedBlocksTable, DELETED_BLOCKS_TABLE);
+ deletedBlocksTable =
+ DELETED_BLOCKS.getTable(this.store);
- validCertsTable = this.store.getTable(VALID_CERTS_TABLE,
- BigInteger.class, X509Certificate.class);
- checkTableStatus(validCertsTable, VALID_CERTS_TABLE);
+ checkTableStatus(deletedBlocksTable,
+ DELETED_BLOCKS.getName());
- revokedCertsTable = this.store.getTable(REVOKED_CERTS_TABLE,
- BigInteger.class, X509Certificate.class);
- checkTableStatus(revokedCertsTable, REVOKED_CERTS_TABLE);
+ validCertsTable = VALID_CERTS.getTable(store);
+
+ checkTableStatus(validCertsTable, VALID_CERTS.getName());
+
+ revokedCertsTable = REVOKED_CERTS.getTable(store);
+
+ checkTableStatus(revokedCertsTable, REVOKED_CERTS.getName());
+
+ pipelineTable = PIPELINES.getTable(store);
+
+ containerTable = CONTAINERS.getTable(store);
}
}
@@ -163,6 +153,21 @@
}
@Override
+ public Table<PipelineID, Pipeline> getPipelineTable() {
+ return pipelineTable;
+ }
+
+ @Override
+ public BatchOperationHandler getBatchHandler() {
+ return this.store;
+ }
+
+ @Override
+ public Table<ContainerID, ContainerInfo> getContainerTable() {
+ return containerTable;
+ }
+
+ @Override
public Long getCurrentTXID() {
return this.txID.get();
}
@@ -174,8 +179,8 @@
* @throws IOException
*/
private Long getLargestRecordedTXID() throws IOException {
- try (TableIterator<Long, DeletedBlocksTransaction> txIter =
- deletedBlocksTable.iterator()) {
+ try (TableIterator<Long, ? extends KeyValue<Long, DeletedBlocksTransaction>>
+ txIter = deletedBlocksTable.iterator()) {
txIter.seekToLast();
Long txid = txIter.key();
if (txid != null) {
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NewNodeHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NewNodeHandler.java
index 1dc924b..a40a63a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NewNodeHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NewNodeHandler.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,7 +18,7 @@
package org.apache.hadoop.hdds.scm.node;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.server.events.EventHandler;
@@ -30,16 +30,17 @@
public class NewNodeHandler implements EventHandler<DatanodeDetails> {
private final PipelineManager pipelineManager;
- private final Configuration conf;
+ private final ConfigurationSource conf;
- public NewNodeHandler(PipelineManager pipelineManager, Configuration conf) {
+ public NewNodeHandler(PipelineManager pipelineManager,
+ ConfigurationSource conf) {
this.pipelineManager = pipelineManager;
this.conf = conf;
}
@Override
public void onMessage(DatanodeDetails datanodeDetails,
- EventPublisher publisher) {
+ EventPublisher publisher) {
pipelineManager.triggerPipelineCreation();
}
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
index cacf077..b6248aa 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
@@ -18,42 +18,46 @@
package org.apache.hadoop.hdds.scm.node;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
-import org.apache.hadoop.hdds.utils.HddsServerUtil;
-import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
-import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
-import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.scm.node.states.*;
-import org.apache.hadoop.hdds.scm.node.states.Node2PipelineMap;
-import org.apache.hadoop.hdds.server.events.Event;
-import org.apache.hadoop.hdds.server.events.EventPublisher;
-import org.apache.hadoop.ozone.common.statemachine
- .InvalidStateTransitionException;
-import org.apache.hadoop.ozone.common.statemachine.StateMachine;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.util.concurrent.HadoopExecutors;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.Closeable;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_DEADNODE_INTERVAL;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_STALENODE_INTERVAL;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.node.states.Node2PipelineMap;
+import org.apache.hadoop.hdds.scm.node.states.NodeAlreadyExistsException;
+import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
+import org.apache.hadoop.hdds.scm.node.states.NodeStateMap;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.hdds.server.events.Event;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.hdds.utils.HddsServerUtil;
+import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException;
+import org.apache.hadoop.ozone.common.statemachine.StateMachine;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* NodeStateManager maintains the state of all the datanodes in the cluster. All
@@ -144,7 +148,8 @@
*
* @param conf Configuration
*/
- public NodeStateManager(Configuration conf, EventPublisher eventPublisher) {
+ public NodeStateManager(ConfigurationSource conf,
+ EventPublisher eventPublisher) {
this.nodeStateMap = new NodeStateMap();
this.node2PipelineMap = new Node2PipelineMap();
this.eventPublisher = eventPublisher;
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NonHealthyToHealthyNodeHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NonHealthyToHealthyNodeHandler.java
index 5976c17..cc32f84 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NonHealthyToHealthyNodeHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NonHealthyToHealthyNodeHandler.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.hdds.scm.node;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
@@ -31,7 +31,7 @@
implements EventHandler<DatanodeDetails> {
private final PipelineManager pipelineManager;
- private final Configuration conf;
+ private final ConfigurationSource conf;
public NonHealthyToHealthyNodeHandler(
PipelineManager pipelineManager, OzoneConfiguration conf) {
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java
index dbb4242..c515f2f 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java
@@ -18,12 +18,6 @@
package org.apache.hadoop.hdds.scm.node;
-import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DEAD;
-import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DECOMMISSIONED;
-import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DECOMMISSIONING;
-import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY;
-import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.STALE;
-
import java.util.Map;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
@@ -39,6 +33,12 @@
import org.apache.hadoop.metrics2.lib.MutableCounterLong;
import org.apache.hadoop.ozone.OzoneConsts;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DEAD;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DECOMMISSIONED;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DECOMMISSIONING;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.STALE;
+
/**
* This class maintains Node related metrics.
*/
@@ -46,7 +46,7 @@
@Metrics(about = "SCM NodeManager Metrics", context = OzoneConsts.OZONE)
public final class SCMNodeMetrics implements MetricsSource {
- private static final String SOURCE_NAME =
+ public static final String SOURCE_NAME =
SCMNodeMetrics.class.getSimpleName();
private @Metric MutableCounterLong numHBProcessed;
@@ -113,14 +113,13 @@
}
/**
- * Get aggregated counter and gauage metrics.
+ * Get aggregated counter and gauge metrics.
*/
@Override
@SuppressWarnings("SuspiciousMethodCalls")
public void getMetrics(MetricsCollector collector, boolean all) {
Map<String, Integer> nodeCount = managerMXBean.getNodeCount();
Map<String, Long> nodeInfo = managerMXBean.getNodeInfo();
-
registry.snapshot(
collector.addRecord(registry.info()) // Add annotated ones first
.addGauge(Interns.info(
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java
index 26e8f5f..5530e73 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.hdds.scm.node;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
@@ -41,7 +41,7 @@
private final NodeManager nodeManager;
private final PipelineManager pipelineManager;
- private final Configuration conf;
+ private final ConfigurationSource conf;
public StaleNodeHandler(NodeManager nodeManager,
PipelineManager pipelineManager, OzoneConfiguration conf) {
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java
index b8f0fb6..f7f1d52 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.hdds.scm.pipeline;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.ozone.OzoneConfigKeys;
@@ -41,11 +41,11 @@
private final Scheduler scheduler;
private final AtomicBoolean isPipelineCreatorRunning;
private final PipelineManager pipelineManager;
- private final Configuration conf;
+ private final ConfigurationSource conf;
private ScheduledFuture<?> periodicTask;
BackgroundPipelineCreator(PipelineManager pipelineManager,
- Scheduler scheduler, Configuration conf) {
+ Scheduler scheduler, ConfigurationSource conf) {
this.pipelineManager = pipelineManager;
this.conf = conf;
this.scheduler = scheduler;
@@ -91,8 +91,13 @@
private boolean skipCreation(HddsProtos.ReplicationFactor factor,
HddsProtos.ReplicationType type,
boolean autoCreate) {
- return factor == HddsProtos.ReplicationFactor.ONE &&
- type == HddsProtos.ReplicationType.RATIS && (!autoCreate);
+ if (type == HddsProtos.ReplicationType.RATIS) {
+ return factor == HddsProtos.ReplicationFactor.ONE && (!autoCreate);
+ } else {
+ // For STAND_ALONE Replication Type, Replication Factor 3 should not be
+ // used.
+ return factor == HddsProtos.ReplicationFactor.THREE;
+ }
}
private void createPipelines() {
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineActionHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineActionHandler.java
index 2f0e224..0720694 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineActionHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineActionHandler.java
@@ -17,13 +17,18 @@
package org.apache.hadoop.hdds.scm.pipeline;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ClosePipelineInfo;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineAction;
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineActionsFromDatanode;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.ozone.protocol.commands.ClosePipelineCommand;
+import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,7 +44,7 @@
LoggerFactory.getLogger(PipelineActionHandler.class);
private final PipelineManager pipelineManager;
- private final Configuration ozoneConf;
+ private final ConfigurationSource ozoneConf;
public PipelineActionHandler(PipelineManager pipelineManager,
OzoneConfiguration conf) {
@@ -50,25 +55,45 @@
@Override
public void onMessage(PipelineActionsFromDatanode report,
EventPublisher publisher) {
- for (PipelineAction action : report.getReport().getPipelineActionsList()) {
- if (action.getAction() == PipelineAction.Action.CLOSE) {
- PipelineID pipelineID = null;
- try {
- pipelineID = PipelineID.
- getFromProtobuf(action.getClosePipeline().getPipelineID());
- Pipeline pipeline = pipelineManager.getPipeline(pipelineID);
- LOG.error("Received pipeline action {} for {} from datanode {}. " +
- "Reason : {}", action.getAction(), pipeline,
- report.getDatanodeDetails(),
- action.getClosePipeline().getDetailedReason());
- pipelineManager.finalizeAndDestroyPipeline(pipeline, true);
- } catch (IOException ioe) {
- LOG.error("Could not execute pipeline action={} pipeline={} {}",
- action, pipelineID, ioe);
- }
+
+ report.getReport().getPipelineActionsList().forEach(action ->
+ processPipelineAction(report.getDatanodeDetails(), action, publisher));
+ }
+
+ /**
+ * Process the given PipelineAction.
+ *
+ * @param datanode the datanode which has sent the PipelineAction
+ * @param pipelineAction the PipelineAction
+ * @param publisher EventPublisher to fire new events if required
+ */
+ private void processPipelineAction(final DatanodeDetails datanode,
+ final PipelineAction pipelineAction,
+ final EventPublisher publisher) {
+ final ClosePipelineInfo info = pipelineAction.getClosePipeline();
+ final PipelineAction.Action action = pipelineAction.getAction();
+ final PipelineID pid = PipelineID.getFromProtobuf(info.getPipelineID());
+ try {
+ LOG.info("Received pipeline action {} for {} from datanode {}. " +
+ "Reason : {}", action, pid, datanode.getUuidString(),
+ info.getDetailedReason());
+
+ if (action == PipelineAction.Action.CLOSE) {
+ pipelineManager.finalizeAndDestroyPipeline(
+ pipelineManager.getPipeline(pid), true);
} else {
- LOG.error("unknown pipeline action:{}", action.getAction());
+ LOG.error("unknown pipeline action:{}", action);
}
+ } catch (PipelineNotFoundException e) {
+ LOG.warn("Pipeline action {} received for unknown pipeline {}, " +
+ "firing close pipeline event.", action, pid);
+ publisher.fireEvent(SCMEvents.DATANODE_COMMAND,
+ new CommandForDatanode<>(datanode.getUuid(),
+ new ClosePipelineCommand(pid)));
+ } catch (IOException ioe) {
+ LOG.error("Could not execute pipeline action={} pipeline={}",
+ action, pid, ioe);
}
}
+
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineFactory.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineFactory.java
index 58a8fd7..e1cf382 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineFactory.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineFactory.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.hdds.scm.pipeline;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
@@ -40,10 +40,10 @@
private Map<ReplicationType, PipelineProvider> providers;
PipelineFactory(NodeManager nodeManager, PipelineStateManager stateManager,
- Configuration conf, EventPublisher eventPublisher) {
+ ConfigurationSource conf, EventPublisher eventPublisher) {
providers = new HashMap<>();
providers.put(ReplicationType.STAND_ALONE,
- new SimplePipelineProvider(nodeManager));
+ new SimplePipelineProvider(nodeManager, stateManager));
providers.put(ReplicationType.RATIS,
new RatisPipelineProvider(nodeManager, stateManager, conf,
eventPublisher));
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java
index 093cdc6..48068d8 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java
@@ -18,23 +18,24 @@
package org.apache.hadoop.hdds.scm.pipeline;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
-import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.hdds.scm.safemode.SafeModeNotification;
-
import java.io.Closeable;
import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.NavigableSet;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager.SafeModeStatus;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+
/**
* Interface which exposes the api for pipeline management.
*/
public interface PipelineManager extends Closeable, PipelineManagerMXBean,
- SafeModeNotification {
+ EventHandler<SafeModeStatus> {
Pipeline createPipeline(ReplicationType type, ReplicationFactor factor)
throws IOException;
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
index e96b120..524b5ec 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
@@ -20,21 +20,19 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.SCMCommonPlacementPolicy;
-import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.net.NetworkTopology;
-import org.apache.hadoop.hdds.scm.net.Node;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
-import java.util.Collection;
+import java.util.Comparator;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
@@ -54,8 +52,9 @@
LoggerFactory.getLogger(PipelinePlacementPolicy.class);
private final NodeManager nodeManager;
private final PipelineStateManager stateManager;
- private final Configuration conf;
+ private final ConfigurationSource conf;
private final int heavyNodeCriteria;
+ private static final int REQUIRED_RACKS = 2;
/**
* Constructs a pipeline placement with considering network topology,
@@ -66,7 +65,7 @@
* @param conf Configuration
*/
public PipelinePlacementPolicy(final NodeManager nodeManager,
- final PipelineStateManager stateManager, final Configuration conf) {
+ final PipelineStateManager stateManager, final ConfigurationSource conf) {
super(nodeManager, conf);
this.nodeManager = nodeManager;
this.conf = conf;
@@ -76,19 +75,8 @@
ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT_DEFAULT);
}
- /**
- * Returns true if this node meets the criteria.
- *
- * @param datanodeDetails DatanodeDetails
- * @param nodesRequired nodes required count
- * @return true if we have enough space.
- */
- @VisibleForTesting
- boolean meetCriteria(DatanodeDetails datanodeDetails, int nodesRequired) {
- if (heavyNodeCriteria == 0) {
- // no limit applied.
- return true;
- }
+ int currentPipelineCount(DatanodeDetails datanodeDetails, int nodesRequired) {
+
// Datanodes from pipeline in some states can also be considered available
// for pipeline allocation. Thus the number of these pipeline shall be
// deducted from total heaviness calculation.
@@ -110,21 +98,16 @@
pipelineNumDeductable++;
}
}
- boolean meet = (nodeManager.getPipelinesCount(datanodeDetails)
- - pipelineNumDeductable) < heavyNodeCriteria;
- if (!meet && LOG.isDebugEnabled()) {
- LOG.debug("Pipeline Placement: can't place more pipeline on heavy " +
- "datanode： " + datanodeDetails.getUuid().toString() +
- " Heaviness: " + nodeManager.getPipelinesCount(datanodeDetails) +
- " limit: " + heavyNodeCriteria);
- }
- return meet;
+ return pipelines.size() - pipelineNumDeductable;
}
+
+
/**
* Filter out viable nodes based on
* 1. nodes that are healthy
* 2. nodes that are not too heavily engaged in other pipelines
+ * The results are sorted based on pipeline count of each node.
*
* @param excludedNodes - excluded nodes
* @param nodesRequired - number of datanodes required.
@@ -154,8 +137,15 @@
// filter nodes that meet the size and pipeline engagement criteria.
// Pipeline placement doesn't take node space left into account.
+ // Sort the DNs by pipeline load.
+ // TODO check if sorting could cause performance issue: HDDS-3466.
List<DatanodeDetails> healthyList = healthyNodes.stream()
- .filter(d -> meetCriteria(d, nodesRequired))
+ .map(d ->
+ new DnWithPipelines(d, currentPipelineCount(d, nodesRequired)))
+ .filter(d ->
+ ((d.getPipelines() < heavyNodeCriteria) || heavyNodeCriteria == 0))
+ .sorted(Comparator.comparingInt(DnWithPipelines::getPipelines))
+ .map(d -> d.getDn())
.collect(Collectors.toList());
if (healthyList.size() < nodesRequired) {
@@ -253,7 +243,7 @@
// Since nodes are widely distributed, the results should be selected
// base on distance in topology, rack awareness and load balancing.
List<DatanodeDetails> exclude = new ArrayList<>();
- // First choose an anchor nodes randomly
+ // First choose an anchor node.
DatanodeDetails anchor = chooseNode(healthyNodes);
if (anchor != null) {
results.add(anchor);
@@ -291,8 +281,9 @@
// Pick remaining nodes based on the existence of rack awareness.
DatanodeDetails pick = null;
if (rackAwareness) {
- pick = chooseNodeFromNetworkTopology(
- nodeManager.getClusterNetworkTopologyMap(), anchor, exclude);
+ pick = chooseNodeBasedOnSameRack(
+ healthyNodes, exclude,
+ nodeManager.getClusterNetworkTopologyMap(), anchor);
}
// fall back protection
if (pick == null) {
@@ -333,24 +324,7 @@
if (healthyNodes == null || healthyNodes.isEmpty()) {
return null;
}
- int firstNodeNdx = getRand().nextInt(healthyNodes.size());
- int secondNodeNdx = getRand().nextInt(healthyNodes.size());
-
- DatanodeDetails datanodeDetails;
- // There is a possibility that both numbers will be same.
- // if that is so, we just return the node.
- if (firstNodeNdx == secondNodeNdx) {
- datanodeDetails = healthyNodes.get(firstNodeNdx);
- } else {
- DatanodeDetails firstNodeDetails = healthyNodes.get(firstNodeNdx);
- DatanodeDetails secondNodeDetails = healthyNodes.get(secondNodeNdx);
- SCMNodeMetric firstNodeMetric =
- nodeManager.getNodeStat(firstNodeDetails);
- SCMNodeMetric secondNodeMetric =
- nodeManager.getNodeStat(secondNodeDetails);
- datanodeDetails = firstNodeMetric.isGreater(secondNodeMetric.get())
- ? firstNodeDetails : secondNodeDetails;
- }
+ DatanodeDetails datanodeDetails = healthyNodes.get(0);
healthyNodes.remove(datanodeDetails);
return datanodeDetails;
}
@@ -373,13 +347,31 @@
return null;
}
- for (DatanodeDetails node : healthyNodes) {
- if (excludedNodes.contains(node) ||
- anchor.getNetworkLocation().equals(node.getNetworkLocation())) {
- continue;
- } else {
- return node;
- }
+ List<DatanodeDetails> nodesOnOtherRack = healthyNodes.stream().filter(
+ p -> !excludedNodes.contains(p)
+ && !anchor.getNetworkLocation().equals(p.getNetworkLocation()))
+ .collect(Collectors.toList());
+ if (!nodesOnOtherRack.isEmpty()) {
+ return nodesOnOtherRack.get(0);
+ }
+ return null;
+ }
+
+ @VisibleForTesting
+ protected DatanodeDetails chooseNodeBasedOnSameRack(
+ List<DatanodeDetails> healthyNodes, List<DatanodeDetails> excludedNodes,
+ NetworkTopology networkTopology, DatanodeDetails anchor) {
+ Preconditions.checkArgument(networkTopology != null);
+ if (checkAllNodesAreEqual(networkTopology)) {
+ return null;
+ }
+
+ List<DatanodeDetails> nodesOnSameRack = healthyNodes.stream().filter(
+ p -> !excludedNodes.contains(p)
+ && anchor.getNetworkLocation().equals(p.getNetworkLocation()))
+ .collect(Collectors.toList());
+ if (!nodesOnSameRack.isEmpty()) {
+ return nodesOnSameRack.get(0);
}
return null;
}
@@ -398,31 +390,27 @@
return (topology.getNumOfNodes(topology.getMaxLevel() - 1) == 1);
}
- /**
- * Choose node based on network topology.
- * @param networkTopology network topology
- * @param anchor anchor datanode to start with
- * @param excludedNodes excluded datanodes
- * @return chosen datanode
- */
- @VisibleForTesting
- protected DatanodeDetails chooseNodeFromNetworkTopology(
- NetworkTopology networkTopology, DatanodeDetails anchor,
- List<DatanodeDetails> excludedNodes) {
- Preconditions.checkArgument(networkTopology != null);
-
- Collection<Node> excluded = new ArrayList<>();
- if (excludedNodes != null && excludedNodes.size() != 0) {
- excluded.addAll(excludedNodes);
- }
-
- Node pick = networkTopology.chooseRandom(
- anchor.getNetworkLocation(), excluded);
- DatanodeDetails pickedNode = (DatanodeDetails) pick;
- if (pickedNode == null) {
- LOG.debug("Pick node is null, excluded nodes {}, anchor {}.",
- excluded, anchor);
- }
- return pickedNode;
+ @Override
+ protected int getRequiredRackCount() {
+ return REQUIRED_RACKS;
}
+
+ private static class DnWithPipelines {
+ private DatanodeDetails dn;
+ private int pipelines;
+
+ DnWithPipelines(DatanodeDetails dn, int pipelines) {
+ this.dn = dn;
+ this.pipelines = pipelines;
+ }
+
+ public int getPipelines() {
+ return pipelines;
+ }
+
+ public DatanodeDetails getDn() {
+ return dn;
+ }
+ }
+
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineProvider.java
index c00ff78..533f77e 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineProvider.java
@@ -18,22 +18,81 @@
package org.apache.hadoop.hdds.scm.pipeline;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
-
import java.io.IOException;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
/**
* Interface for creating pipelines.
*/
-public interface PipelineProvider {
+public abstract class PipelineProvider {
- Pipeline create(ReplicationFactor factor) throws IOException;
+ private final NodeManager nodeManager;
+ private final PipelineStateManager stateManager;
- Pipeline create(ReplicationFactor factor, List<DatanodeDetails> nodes);
+ public PipelineProvider(NodeManager nodeManager,
+ PipelineStateManager stateManager) {
+ this.nodeManager = nodeManager;
+ this.stateManager = stateManager;
+ }
- void close(Pipeline pipeline) throws IOException;
+ public PipelineProvider() {
+ this.nodeManager = null;
+ this.stateManager = null;
+ }
- void shutdown();
+ public NodeManager getNodeManager() {
+ return nodeManager;
+ }
+
+ public PipelineStateManager getPipelineStateManager() {
+ return stateManager;
+ }
+
+ protected abstract Pipeline create(ReplicationFactor factor)
+ throws IOException;
+
+ protected abstract Pipeline create(ReplicationFactor factor,
+ List<DatanodeDetails> nodes);
+
+ protected abstract void close(Pipeline pipeline) throws IOException;
+
+ protected abstract void shutdown();
+
+ List<DatanodeDetails> pickNodesNeverUsed(ReplicationType type,
+ ReplicationFactor factor) throws SCMException {
+ Set<DatanodeDetails> dnsUsed = new HashSet<>();
+ stateManager.getPipelines(type, factor).stream().filter(
+ p -> p.getPipelineState().equals(Pipeline.PipelineState.OPEN) ||
+ p.getPipelineState().equals(Pipeline.PipelineState.DORMANT) ||
+ p.getPipelineState().equals(Pipeline.PipelineState.ALLOCATED))
+ .forEach(p -> dnsUsed.addAll(p.getNodes()));
+
+ // Get list of healthy nodes
+ List<DatanodeDetails> dns = nodeManager
+ .getNodes(HddsProtos.NodeState.HEALTHY)
+ .parallelStream()
+ .filter(dn -> !dnsUsed.contains(dn))
+ .limit(factor.getNumber())
+ .collect(Collectors.toList());
+ if (dns.size() < factor.getNumber()) {
+ String e = String
+ .format("Cannot create pipeline of factor %d using %d nodes." +
+ " Used %d nodes. Healthy nodes %d", factor.getNumber(),
+ dns.size(), dnsUsed.size(),
+ nodeManager.getNodes(HddsProtos.NodeState.HEALTHY).size());
+ throw new SCMException(e,
+ SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
+ }
+ return dns;
+ }
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java
index 1a93f22..f45b3a9 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java
@@ -20,7 +20,7 @@
import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -50,13 +50,13 @@
private static final Logger LOGGER = LoggerFactory.getLogger(
PipelineReportHandler.class);
private final PipelineManager pipelineManager;
- private final Configuration conf;
+ private final ConfigurationSource conf;
private final SafeModeManager scmSafeModeManager;
private final boolean pipelineAvailabilityCheck;
private final SCMPipelineMetrics metrics;
public PipelineReportHandler(SafeModeManager scmSafeModeManager,
- PipelineManager pipelineManager, Configuration conf) {
+ PipelineManager pipelineManager, ConfigurationSource conf) {
Preconditions.checkNotNull(pipelineManager);
this.scmSafeModeManager = scmSafeModeManager;
this.pipelineManager = pipelineManager;
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateMap.java
index 8e0f32d..df61320 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateMap.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateMap.java
@@ -211,7 +211,8 @@
if (state == PipelineState.OPEN) {
return Collections.unmodifiableList(
- query2OpenPipelines.get(new PipelineQuery(type, factor)));
+ query2OpenPipelines.getOrDefault(
+ new PipelineQuery(type, factor), Collections.EMPTY_LIST));
}
return pipelineMap.values().stream().filter(
pipeline -> pipeline.getType() == type
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
index ad100ab..4d91541 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
@@ -18,11 +18,14 @@
package org.apache.hadoop.hdds.scm.pipeline;
-import org.apache.hadoop.conf.Configuration;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
@@ -32,36 +35,28 @@
import org.apache.hadoop.ozone.protocol.commands.ClosePipelineCommand;
import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
import org.apache.hadoop.ozone.protocol.commands.CreatePipelineCommand;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
/**
* Implements Api for creating ratis pipelines.
*/
-public class RatisPipelineProvider implements PipelineProvider {
+public class RatisPipelineProvider extends PipelineProvider {
private static final Logger LOG =
LoggerFactory.getLogger(RatisPipelineProvider.class);
- private final NodeManager nodeManager;
- private final PipelineStateManager stateManager;
- private final Configuration conf;
+ private final ConfigurationSource conf;
private final EventPublisher eventPublisher;
private final PipelinePlacementPolicy placementPolicy;
private int pipelineNumberLimit;
private int maxPipelinePerDatanode;
RatisPipelineProvider(NodeManager nodeManager,
- PipelineStateManager stateManager, Configuration conf,
+ PipelineStateManager stateManager, ConfigurationSource conf,
EventPublisher eventPublisher) {
- this.nodeManager = nodeManager;
- this.stateManager = stateManager;
+ super(nodeManager, stateManager);
this.conf = conf;
this.eventPublisher = eventPublisher;
this.placementPolicy =
@@ -74,35 +69,6 @@
ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT_DEFAULT);
}
- private List<DatanodeDetails> pickNodesNeverUsed(ReplicationFactor factor)
- throws SCMException {
- Set<DatanodeDetails> dnsUsed = new HashSet<>();
- stateManager.getPipelines(ReplicationType.RATIS, factor)
- .stream().filter(
- p -> p.getPipelineState().equals(PipelineState.OPEN) ||
- p.getPipelineState().equals(PipelineState.DORMANT) ||
- p.getPipelineState().equals(PipelineState.ALLOCATED))
- .forEach(p -> dnsUsed.addAll(p.getNodes()));
-
- // Get list of healthy nodes
- List<DatanodeDetails> dns = nodeManager
- .getNodes(HddsProtos.NodeState.HEALTHY)
- .parallelStream()
- .filter(dn -> !dnsUsed.contains(dn))
- .limit(factor.getNumber())
- .collect(Collectors.toList());
- if (dns.size() < factor.getNumber()) {
- String e = String
- .format("Cannot create pipeline of factor %d using %d nodes." +
- " Used %d nodes. Healthy nodes %d", factor.getNumber(),
- dns.size(), dnsUsed.size(),
- nodeManager.getNodes(HddsProtos.NodeState.HEALTHY).size());
- throw new SCMException(e,
- SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
- }
- return dns;
- }
-
private boolean exceedPipelineNumberLimit(ReplicationFactor factor) {
if (factor != ReplicationFactor.THREE) {
// Only put limits for Factor THREE pipelines.
@@ -110,20 +76,22 @@
}
// Per datanode limit
if (maxPipelinePerDatanode > 0) {
- return (stateManager.getPipelines(ReplicationType.RATIS, factor).size() -
- stateManager.getPipelines(ReplicationType.RATIS, factor,
- Pipeline.PipelineState.CLOSED).size()) > maxPipelinePerDatanode *
- nodeManager.getNodeCount(HddsProtos.NodeState.HEALTHY) /
+ return (getPipelineStateManager().getPipelines(
+ ReplicationType.RATIS, factor).size() -
+ getPipelineStateManager().getPipelines(ReplicationType.RATIS, factor,
+ PipelineState.CLOSED).size()) > maxPipelinePerDatanode *
+ getNodeManager().getNodeCount(HddsProtos.NodeState.HEALTHY) /
factor.getNumber();
}
// Global limit
if (pipelineNumberLimit > 0) {
- return (stateManager.getPipelines(ReplicationType.RATIS,
- ReplicationFactor.THREE).size() - stateManager.getPipelines(
- ReplicationType.RATIS, ReplicationFactor.THREE,
- Pipeline.PipelineState.CLOSED).size()) >
- (pipelineNumberLimit - stateManager.getPipelines(
+ return (getPipelineStateManager().getPipelines(ReplicationType.RATIS,
+ ReplicationFactor.THREE).size() -
+ getPipelineStateManager().getPipelines(
+ ReplicationType.RATIS, ReplicationFactor.THREE,
+ PipelineState.CLOSED).size()) >
+ (pipelineNumberLimit - getPipelineStateManager().getPipelines(
ReplicationType.RATIS, ReplicationFactor.ONE).size());
}
@@ -143,7 +111,7 @@
switch(factor) {
case ONE:
- dns = pickNodesNeverUsed(ReplicationFactor.ONE);
+ dns = pickNodesNeverUsed(ReplicationType.RATIS, ReplicationFactor.ONE);
break;
case THREE:
dns = placementPolicy.chooseDatanodes(null,
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java
index db9260e..5c9b202 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineUtils.java
@@ -21,11 +21,12 @@
import java.util.List;
import java.util.stream.Collectors;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.ratis.RatisHelper;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+
import org.apache.ratis.client.RaftClient;
import org.apache.ratis.grpc.GrpcTlsConfig;
import org.apache.ratis.protocol.RaftGroup;
@@ -56,7 +57,8 @@
* @param grpcTlsConfig
* @throws IOException
*/
- public static void destroyPipeline(Pipeline pipeline, Configuration ozoneConf,
+ public static void destroyPipeline(Pipeline pipeline,
+ ConfigurationSource ozoneConf,
GrpcTlsConfig grpcTlsConfig) {
final RaftGroup group = RatisHelper.newRaftGroup(pipeline);
if (LOG.isDebugEnabled()) {
@@ -82,7 +84,8 @@
* @throws IOException
*/
static void destroyPipeline(DatanodeDetails dn, PipelineID pipelineID,
- Configuration ozoneConf, GrpcTlsConfig grpcTlsConfig) throws IOException {
+ ConfigurationSource ozoneConf, GrpcTlsConfig grpcTlsConfig)
+ throws IOException {
final String rpcType = ozoneConf
.get(ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY,
ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT);
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
index 83bc902..941ce19 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
@@ -18,49 +18,43 @@
package org.apache.hadoop.hdds.scm.pipeline;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager;
-import org.apache.hadoop.hdds.server.ServerUtils;
-import org.apache.hadoop.hdds.server.events.EventPublisher;
-import org.apache.hadoop.metrics2.util.MBeans;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.hdds.utils.MetadataKeyFilters;
-import org.apache.hadoop.hdds.utils.MetadataStore;
-import org.apache.hadoop.hdds.utils.MetadataStoreBuilder;
-import org.apache.hadoop.hdds.utils.Scheduler;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import javax.management.ObjectName;
-import java.io.File;
import java.io.IOException;
import java.time.Duration;
import java.time.Instant;
+import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.NavigableSet;
import java.util.Set;
-import java.util.Collection;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors;
-import static org.apache.hadoop.ozone.OzoneConsts.SCM_PIPELINE_DB;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager.SafeModeStatus;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.hdds.utils.Scheduler;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.Table.KeyValue;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.metrics2.util.MBeans;
+import org.apache.hadoop.util.Time;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Implements api needed for management of pipelines. All the write operations
@@ -77,34 +71,44 @@
private PipelineStateManager stateManager;
private final BackgroundPipelineCreator backgroundPipelineCreator;
private Scheduler scheduler;
- private MetadataStore pipelineStore;
private final EventPublisher eventPublisher;
private final NodeManager nodeManager;
private final SCMPipelineMetrics metrics;
- private final Configuration conf;
+ private final ConfigurationSource conf;
private long pipelineWaitDefaultTimeout;
// Pipeline Manager MXBean
private ObjectName pmInfoBean;
- private final AtomicBoolean isInSafeMode;
+ private Table<PipelineID, Pipeline> pipelineStore;
- public SCMPipelineManager(Configuration conf, NodeManager nodeManager,
+ private final AtomicBoolean isInSafeMode;
+ // Used to track if the safemode pre-checks have completed. This is designed
+ // to prevent pipelines being created until sufficient nodes have registered.
+ private final AtomicBoolean pipelineCreationAllowed;
+
+ public SCMPipelineManager(ConfigurationSource conf,
+ NodeManager nodeManager,
+ Table<PipelineID, Pipeline> pipelineStore,
EventPublisher eventPublisher)
throws IOException {
- this(conf, nodeManager, eventPublisher, null, null);
+ this(conf, nodeManager, pipelineStore, eventPublisher, null, null);
this.stateManager = new PipelineStateManager();
this.pipelineFactory = new PipelineFactory(nodeManager,
stateManager, conf, eventPublisher);
+ this.pipelineStore = pipelineStore;
initializePipelineState();
}
- protected SCMPipelineManager(Configuration conf, NodeManager nodeManager,
- EventPublisher eventPublisher,
- PipelineStateManager pipelineStateManager,
- PipelineFactory pipelineFactory)
+ protected SCMPipelineManager(ConfigurationSource conf,
+ NodeManager nodeManager,
+ Table<PipelineID, Pipeline> pipelineStore,
+ EventPublisher eventPublisher,
+ PipelineStateManager pipelineStateManager,
+ PipelineFactory pipelineFactory)
throws IOException {
this.lock = new ReentrantReadWriteLock();
+ this.pipelineStore = pipelineStore;
this.conf = conf;
this.pipelineFactory = pipelineFactory;
this.stateManager = pipelineStateManager;
@@ -112,16 +116,6 @@
scheduler = new Scheduler("RatisPipelineUtilsThread", false, 1);
this.backgroundPipelineCreator =
new BackgroundPipelineCreator(this, scheduler, conf);
- int cacheSize = conf.getInt(ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_MB,
- ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_DEFAULT);
- final File pipelineDBPath = getPipelineDBPath(conf);
- this.pipelineStore =
- MetadataStoreBuilder.newBuilder()
- .setCreateIfMissing(true)
- .setConf(conf)
- .setDbFile(pipelineDBPath)
- .setCacheSize(cacheSize * OzoneConsts.MB)
- .build();
this.eventPublisher = eventPublisher;
this.nodeManager = nodeManager;
this.metrics = SCMPipelineMetrics.create();
@@ -134,6 +128,9 @@
this.isInSafeMode = new AtomicBoolean(conf.getBoolean(
HddsConfigKeys.HDDS_SCM_SAFEMODE_ENABLED,
HddsConfigKeys.HDDS_SCM_SAFEMODE_ENABLED_DEFAULT));
+ // Pipeline creation is only allowed after the safemode prechecks have
+ // passed, eg sufficient nodes have registered.
+ this.pipelineCreationAllowed = new AtomicBoolean(!this.isInSafeMode.get());
}
public PipelineStateManager getStateManager() {
@@ -146,23 +143,25 @@
pipelineFactory.setProvider(replicationType, provider);
}
+ @VisibleForTesting
+ public void allowPipelineCreation() {
+ this.pipelineCreationAllowed.set(true);
+ }
+
+ @VisibleForTesting
+ public boolean isPipelineCreationAllowed() {
+ return pipelineCreationAllowed.get();
+ }
+
protected void initializePipelineState() throws IOException {
if (pipelineStore.isEmpty()) {
LOG.info("No pipeline exists in current db");
return;
}
- List<Map.Entry<byte[], byte[]>> pipelines =
- pipelineStore.getSequentialRangeKVs(null, Integer.MAX_VALUE,
- (MetadataKeyFilters.MetadataKeyFilter[])null);
-
- for (Map.Entry<byte[], byte[]> entry : pipelines) {
- HddsProtos.Pipeline.Builder pipelineBuilder = HddsProtos.Pipeline
- .newBuilder(HddsProtos.Pipeline.PARSER.parseFrom(entry.getValue()));
- Pipeline pipeline = Pipeline.getFromProtobuf(pipelineBuilder.setState(
- HddsProtos.PipelineState.PIPELINE_ALLOCATED).build());
- // When SCM is restarted, set Creation time with current time.
- pipeline.setCreationTimestamp(Instant.now());
- Preconditions.checkNotNull(pipeline);
+ TableIterator<PipelineID, ? extends KeyValue<PipelineID, Pipeline>>
+ iterator = pipelineStore.iterator();
+ while (iterator.hasNext()) {
+ Pipeline pipeline = iterator.next().getValue();
stateManager.addPipeline(pipeline);
nodeManager.addPipeline(pipeline);
}
@@ -205,16 +204,23 @@
@Override
public synchronized Pipeline createPipeline(ReplicationType type,
ReplicationFactor factor) throws IOException {
+ if (!isPipelineCreationAllowed() && factor != ReplicationFactor.ONE) {
+ LOG.debug("Pipeline creation is not allowed until safe mode prechecks " +
+ "complete");
+ throw new IOException("Pipeline creation is not allowed as safe mode " +
+ "prechecks have not yet passed");
+ }
lock.writeLock().lock();
try {
Pipeline pipeline = pipelineFactory.create(type, factor);
- pipelineStore.put(pipeline.getId().getProtobuf().toByteArray(),
- pipeline.getProtobufMessage().toByteArray());
+ pipelineStore.put(pipeline.getId(), pipeline);
stateManager.addPipeline(pipeline);
nodeManager.addPipeline(pipeline);
recordMetricsForPipeline(pipeline);
return pipeline;
} catch (IOException ex) {
+ LOG.error("Failed to create pipeline of type {} and factor {}. " +
+ "Exception: {}", type, factor, ex.getMessage());
metrics.incNumPipelineCreationFailed();
throw ex;
} finally {
@@ -563,11 +569,10 @@
* @throws IOException
*/
protected void removePipeline(PipelineID pipelineId) throws IOException {
- byte[] key = pipelineId.getProtobuf().toByteArray();
lock.writeLock().lock();
try {
if (pipelineStore != null) {
- pipelineStore.delete(key);
+ pipelineStore.delete(pipelineId);
Pipeline pipeline = stateManager.removePipeline(pipelineId);
nodeManager.removePipeline(pipeline);
metrics.incNumPipelineDestroyed();
@@ -592,32 +597,17 @@
scheduler = null;
}
- lock.writeLock().lock();
- try {
- if (pipelineStore != null) {
- pipelineStore.close();
- pipelineStore = null;
- }
- } finally {
- lock.writeLock().unlock();
- }
-
if(pmInfoBean != null) {
MBeans.unregister(this.pmInfoBean);
pmInfoBean = null;
}
- if(metrics != null) {
- metrics.unRegister();
- }
+
+ SCMPipelineMetrics.unRegister();
+
// shutdown pipeline provider.
pipelineFactory.shutdown();
}
- protected File getPipelineDBPath(Configuration configuration) {
- File metaDir = ServerUtils.getScmDbDir(configuration);
- return new File(metaDir, SCM_PIPELINE_DB);
- }
-
protected ReadWriteLock getLock() {
return lock;
}
@@ -627,10 +617,6 @@
return pipelineFactory;
}
- protected MetadataStore getPipelineStore() {
- return pipelineStore;
- }
-
protected NodeManager getNodeManager() {
return nodeManager;
}
@@ -640,16 +626,27 @@
return this.isInSafeMode.get();
}
- @Override
- public void handleSafeModeTransition(
- SCMSafeModeManager.SafeModeStatus status) {
- this.isInSafeMode.set(status.getSafeModeStatus());
- if (!status.getSafeModeStatus()) {
- // TODO: #CLUTIL if we reenter safe mode the fixed interval pipeline
- // creation job needs to stop
- startPipelineCreator();
- triggerPipelineCreation();
- }
+ public Table<PipelineID, Pipeline> getPipelineStore() {
+ return pipelineStore;
}
+ @Override
+ public void onMessage(SafeModeStatus status,
+ EventPublisher publisher) {
+ // TODO: #CLUTIL - handle safemode getting re-enabled
+ boolean currentAllowPipelines =
+ pipelineCreationAllowed.getAndSet(status.isPreCheckComplete());
+ boolean currentlyInSafeMode =
+ isInSafeMode.getAndSet(status.isInSafeMode());
+
+ // Trigger pipeline creation only if the preCheck status has changed to
+ // complete.
+ if (isPipelineCreationAllowed() && !currentAllowPipelines) {
+ triggerPipelineCreation();
+ }
+ // Start the pipeline creation thread only when safemode switches off
+ if (!getSafeModeStatus() && currentlyInSafeMode) {
+ startPipelineCreator();
+ }
+ }
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineMetrics.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineMetrics.java
index 0cc60f9..c6424dc 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineMetrics.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineMetrics.java
@@ -46,6 +46,7 @@
SCMPipelineMetrics.class.getSimpleName();
private MetricsRegistry registry;
+ private static SCMPipelineMetrics instance;
private @Metric MutableCounterLong numPipelineAllocated;
private @Metric MutableCounterLong numPipelineCreated;
@@ -70,16 +71,21 @@
*
* @return SCMPipelineMetrics
*/
- public static SCMPipelineMetrics create() {
+ public static synchronized SCMPipelineMetrics create() {
+ if (instance != null) {
+ return instance;
+ }
MetricsSystem ms = DefaultMetricsSystem.instance();
- return ms.register(SOURCE_NAME, "SCM PipelineManager Metrics",
+ instance = ms.register(SOURCE_NAME, "SCM PipelineManager Metrics",
new SCMPipelineMetrics());
+ return instance;
}
/**
* Unregister the metrics instance.
*/
- public void unRegister() {
+ public static void unRegister() {
+ instance = null;
MetricsSystem ms = DefaultMetricsSystem.instance();
ms.unregisterSource(SOURCE_NAME);
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java
index a772a97..c7b6305 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java
@@ -21,7 +21,6 @@
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline.PipelineState;
@@ -32,18 +31,18 @@
/**
* Implements Api for creating stand alone pipelines.
*/
-public class SimplePipelineProvider implements PipelineProvider {
+public class SimplePipelineProvider extends PipelineProvider {
- private final NodeManager nodeManager;
-
- public SimplePipelineProvider(NodeManager nodeManager) {
- this.nodeManager = nodeManager;
+ public SimplePipelineProvider(NodeManager nodeManager,
+ PipelineStateManager stateManager) {
+ super(nodeManager, stateManager);
}
@Override
public Pipeline create(ReplicationFactor factor) throws IOException {
- List<DatanodeDetails> dns =
- nodeManager.getNodes(NodeState.HEALTHY);
+ List<DatanodeDetails> dns = pickNodesNeverUsed(ReplicationType.STAND_ALONE,
+ factor);
+
if (dns.size() < factor.getNumber()) {
String e = String
.format("Cannot create pipeline of factor %d using %d nodes.",
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java
index 851fa35..fb07351 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java
@@ -47,6 +47,7 @@
import org.apache.hadoop.ozone.common.DeleteBlockGroupResult;
import org.apache.hadoop.hdds.utils.ProtocolMessageMetrics;
+import com.google.protobuf.ProtocolMessageEnum;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
import org.slf4j.Logger;
@@ -77,7 +78,8 @@
*/
public ScmBlockLocationProtocolServerSideTranslatorPB(
ScmBlockLocationProtocol impl,
- ProtocolMessageMetrics metrics) throws IOException {
+ ProtocolMessageMetrics<ProtocolMessageEnum> metrics)
+ throws IOException {
this.impl = impl;
dispatcher = new OzoneProtocolMessageDispatcher<>(
"BlockLocationProtocol", metrics, LOG);
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
index cb3a5b3..bc2208d 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
@@ -38,6 +38,8 @@
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerWithPipelineRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerWithPipelineResponseProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerWithPipelineBatchRequestProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerWithPipelineBatchResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetPipelineRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetPipelineResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.InSafeModeRequestProto;
@@ -68,6 +70,7 @@
import org.apache.hadoop.hdds.server.OzoneProtocolMessageDispatcher;
import org.apache.hadoop.hdds.utils.ProtocolMessageMetrics;
+import com.google.protobuf.ProtocolMessageEnum;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
import org.slf4j.Logger;
@@ -104,7 +107,8 @@
*/
public StorageContainerLocationProtocolServerSideTranslatorPB(
StorageContainerLocationProtocol impl,
- ProtocolMessageMetrics protocolMetrics) throws IOException {
+ ProtocolMessageMetrics<ProtocolMessageEnum> protocolMetrics)
+ throws IOException {
this.impl = impl;
this.dispatcher =
new OzoneProtocolMessageDispatcher<>("ScmContainerLocation",
@@ -144,6 +148,14 @@
.setGetContainerWithPipelineResponse(getContainerWithPipeline(
request.getGetContainerWithPipelineRequest()))
.build();
+ case GetContainerWithPipelineBatch:
+ return ScmContainerLocationResponse.newBuilder()
+ .setCmdType(request.getCmdType())
+ .setStatus(Status.OK)
+ .setGetContainerWithPipelineBatchResponse(
+ getContainerWithPipelineBatch(
+ request.getGetContainerWithPipelineBatchRequest()))
+ .build();
case ListContainer:
return ScmContainerLocationResponse.newBuilder()
.setCmdType(request.getCmdType())
@@ -287,6 +299,19 @@
.build();
}
+ public GetContainerWithPipelineBatchResponseProto
+ getContainerWithPipelineBatch(
+ GetContainerWithPipelineBatchRequestProto request) throws IOException {
+ List<ContainerWithPipeline> containers = impl
+ .getContainerWithPipelineBatch(request.getContainerIDsList());
+ GetContainerWithPipelineBatchResponseProto.Builder builder =
+ GetContainerWithPipelineBatchResponseProto.newBuilder();
+ for (ContainerWithPipeline container : containers) {
+ builder.addContainerWithPipelines(container.getProtobuf());
+ }
+ return builder.build();
+ }
+
public SCMListContainerResponseProto listContainer(
SCMListContainerRequestProto request) throws IOException {
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/ContainerSafeModeRule.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/ContainerSafeModeRule.java
index 8eadeb3..2d7466e 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/ContainerSafeModeRule.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/ContainerSafeModeRule.java
@@ -24,7 +24,7 @@
import java.util.concurrent.atomic.AtomicLong;
import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
@@ -51,7 +51,7 @@
private AtomicLong containerWithMinReplicas = new AtomicLong(0);
public ContainerSafeModeRule(String ruleName, EventQueue eventQueue,
- Configuration conf,
+ ConfigurationSource conf,
List<ContainerInfo> containers, SCMSafeModeManager manager) {
super(manager, ruleName, eventQueue);
safeModeCutoff = conf.getDouble(
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/DataNodeSafeModeRule.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/DataNodeSafeModeRule.java
index 1029d71..0afbd27 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/DataNodeSafeModeRule.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/DataNodeSafeModeRule.java
@@ -20,7 +20,7 @@
import java.util.HashSet;
import java.util.UUID;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer.NodeRegistrationContainerReport;
@@ -42,7 +42,7 @@
private HashSet<UUID> registeredDnSet;
public DataNodeSafeModeRule(String ruleName, EventQueue eventQueue,
- Configuration conf,
+ ConfigurationSource conf,
SCMSafeModeManager manager) {
super(manager, ruleName, eventQueue);
requiredDns = conf.getInt(
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/HealthyPipelineSafeModeRule.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/HealthyPipelineSafeModeRule.java
index 688125f..bd58a06 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/HealthyPipelineSafeModeRule.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/HealthyPipelineSafeModeRule.java
@@ -19,20 +19,21 @@
import java.util.HashSet;
import java.util.Set;
-import org.apache.hadoop.conf.Configuration;
+
import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.hdds.server.events.TypedEvent;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Class defining Safe mode exit criteria for Pipelines.
@@ -52,7 +53,7 @@
HealthyPipelineSafeModeRule(String ruleName, EventQueue eventQueue,
PipelineManager pipelineManager,
- SCMSafeModeManager manager, Configuration configuration) {
+ SCMSafeModeManager manager, ConfigurationSource configuration) {
super(manager, ruleName, eventQueue);
healthyPipelinesPercent =
configuration.getDouble(HddsConfigKeys.
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/OneReplicaPipelineSafeModeRule.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/OneReplicaPipelineSafeModeRule.java
index 0783d02..bce4af5 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/OneReplicaPipelineSafeModeRule.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/OneReplicaPipelineSafeModeRule.java
@@ -19,7 +19,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
@@ -54,7 +54,7 @@
public OneReplicaPipelineSafeModeRule(String ruleName, EventQueue eventQueue,
PipelineManager pipelineManager,
- SCMSafeModeManager safeModeManager, Configuration configuration) {
+ SCMSafeModeManager safeModeManager, ConfigurationSource configuration) {
super(safeModeManager, ruleName, eventQueue);
double percent =
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SCMSafeModeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SCMSafeModeManager.java
index 1a309ce..e8f0ab6 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SCMSafeModeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SCMSafeModeManager.java
@@ -17,21 +17,23 @@
*/
package org.apache.hadoop.hdds.scm.safemode;
-import com.google.common.annotations.VisibleForTesting;
-
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.hadoop.conf.Configuration;
+
import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.EventQueue;
+
+import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -80,10 +82,13 @@
private static final Logger LOG =
LoggerFactory.getLogger(SCMSafeModeManager.class);
private final boolean isSafeModeEnabled;
+ private final long waitTime;
private AtomicBoolean inSafeMode = new AtomicBoolean(true);
+ private AtomicBoolean preCheckComplete = new AtomicBoolean(false);
private Map<String, SafeModeExitRule> exitRules = new HashMap(1);
- private Configuration config;
+ private Set<String> preCheckRules = new HashSet<>(1);
+ private ConfigurationSource config;
private static final String CONT_EXIT_RULE = "ContainerSafeModeRule";
private static final String DN_EXIT_RULE = "DataNodeSafeModeRule";
private static final String HEALTHY_PIPELINE_EXIT_RULE =
@@ -92,13 +97,14 @@
"AtleastOneDatanodeReportedRule";
private Set<String> validatedRules = new HashSet<>();
+ private Set<String> validatedPreCheckRules = new HashSet<>(1);
private final EventQueue eventPublisher;
private final PipelineManager pipelineManager;
private final SafeModeMetrics safeModeMetrics;
- public SCMSafeModeManager(Configuration conf,
+ public SCMSafeModeManager(ConfigurationSource conf,
List<ContainerInfo> allContainers, PipelineManager pipelineManager,
EventQueue eventQueue) {
this.config = conf;
@@ -109,6 +115,11 @@
HddsConfigKeys.HDDS_SCM_SAFEMODE_ENABLED_DEFAULT);
+ this.waitTime = conf.getTimeDuration(
+ HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT,
+ HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT_DEFAULT,
+ TimeUnit.MILLISECONDS);
+
if (isSafeModeEnabled) {
this.safeModeMetrics = SafeModeMetrics.create();
ContainerSafeModeRule containerSafeModeRule =
@@ -118,6 +129,7 @@
new DataNodeSafeModeRule(DN_EXIT_RULE, eventQueue, config, this);
exitRules.put(CONT_EXIT_RULE, containerSafeModeRule);
exitRules.put(DN_EXIT_RULE, dataNodeSafeModeRule);
+ preCheckRules.add(DN_EXIT_RULE);
if (conf.getBoolean(
HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK,
HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK_DEFAULT)
@@ -162,8 +174,31 @@
*/
@VisibleForTesting
public void emitSafeModeStatus() {
+ SafeModeStatus safeModeStatus =
+ new SafeModeStatus(getInSafeMode(), getPreCheckComplete());
eventPublisher.fireEvent(SCMEvents.SAFE_MODE_STATUS,
- new SafeModeStatus(getInSafeMode()));
+ safeModeStatus);
+
+ // Only notify the delayed listeners if safemode remains on, as precheck
+ // may have completed.
+ if (safeModeStatus.isInSafeMode()) {
+ eventPublisher.fireEvent(SCMEvents.DELAYED_SAFE_MODE_STATUS,
+ safeModeStatus);
+ } else {
+ // If safemode is off, then notify the delayed listeners with a delay.
+ final Thread safeModeExitThread = new Thread(() -> {
+ try {
+ Thread.sleep(waitTime);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ eventPublisher.fireEvent(SCMEvents.DELAYED_SAFE_MODE_STATUS,
+ safeModeStatus);
+ });
+
+ safeModeExitThread.setDaemon(true);
+ safeModeExitThread.start();
+ }
}
@@ -172,12 +207,20 @@
if (exitRules.get(ruleName) != null) {
validatedRules.add(ruleName);
+ if (preCheckRules.contains(ruleName)) {
+ validatedPreCheckRules.add(ruleName);
+ }
LOG.info("{} rule is successfully validated", ruleName);
} else {
// This should never happen
LOG.error("No Such Exit rule {}", ruleName);
}
+ if (!getPreCheckComplete()) {
+ if (validatedPreCheckRules.size() == preCheckRules.size()) {
+ completePreCheck(eventQueue);
+ }
+ }
if (validatedRules.size() == exitRules.size()) {
// All rules are satisfied, we can exit safe mode.
@@ -188,6 +231,19 @@
}
/**
+ * When all the precheck rules have been validated, set preCheckComplete to
+ * true and then emit the safemode status so any listeners get notified of
+ * the safemode state change.
+ * @param eventQueue
+ */
+ @VisibleForTesting
+ public void completePreCheck(EventPublisher eventQueue) {
+ LOG.info("All SCM safe mode pre check rules have passed");
+ setPreCheckComplete(true);
+ emitSafeModeStatus();
+ }
+
+ /**
* Exit safe mode. It does following actions:
* 1. Set safe mode status to false.
* 2. Emits START_REPLICATION for ReplicationManager.
@@ -198,6 +254,9 @@
@VisibleForTesting
public void exitSafeMode(EventPublisher eventQueue) {
LOG.info("SCM exiting safe mode.");
+ // If safemode is exiting, then pre check must also have passed so
+ // set it to true.
+ setPreCheckComplete(true);
setInSafeMode(false);
// TODO: Remove handler registration as there is no need to listen to
@@ -213,6 +272,10 @@
return inSafeMode.get();
}
+ public boolean getPreCheckComplete() {
+ return preCheckComplete.get();
+ }
+
/**
* Set safe mode status.
*/
@@ -220,6 +283,10 @@
this.inSafeMode.set(inSafeMode);
}
+ public void setPreCheckComplete(boolean newState) {
+ this.preCheckComplete.set(newState);
+ }
+
public static Logger getLogger() {
return LOG;
}
@@ -249,13 +316,20 @@
public static class SafeModeStatus {
private boolean safeModeStatus;
- public SafeModeStatus(boolean safeModeState) {
+ private boolean preCheckPassed;
+
+ public SafeModeStatus(boolean safeModeState, boolean preCheckPassed) {
this.safeModeStatus = safeModeState;
+ this.preCheckPassed = preCheckPassed;
}
- public boolean getSafeModeStatus() {
+ public boolean isInSafeMode() {
return safeModeStatus;
}
+
+ public boolean isPreCheckComplete() {
+ return preCheckPassed;
+ }
}
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeHandler.java
deleted file mode 100644
index 9cbbaa9..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeHandler.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * <p>http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * <p>Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.safemode;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager.SafeModeStatus;
-import org.apache.hadoop.hdds.server.events.EventHandler;
-import org.apache.hadoop.hdds.server.events.EventPublisher;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * Class to handle the activities needed to be performed after exiting safe
- * mode.
- */
-public class SafeModeHandler implements EventHandler<SafeModeStatus> {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(SafeModeHandler.class);
-
- private final long waitTime;
- private final AtomicBoolean isInSafeMode = new AtomicBoolean(true);
- private final List<SafeModeNotification> immediate = new ArrayList<>();
- private final List<SafeModeNotification> delayed = new ArrayList<>();
-
- /**
- * SafeModeHandler, to handle the logic once we exit safe mode.
- * @param configuration
- */
- public SafeModeHandler(Configuration configuration) {
- this.waitTime = configuration.getTimeDuration(
- HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT,
- HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT_DEFAULT,
- TimeUnit.MILLISECONDS);
-
- final boolean safeModeEnabled = configuration.getBoolean(
- HddsConfigKeys.HDDS_SCM_SAFEMODE_ENABLED,
- HddsConfigKeys.HDDS_SCM_SAFEMODE_ENABLED_DEFAULT);
- isInSafeMode.set(safeModeEnabled);
- }
-
- /**
- * Add any objects which should be notified immediately on a safemode status
- * change.
- * @param objs List of objects to be notified.
- */
- public void notifyImmediately(SafeModeNotification...objs) {
- for (SafeModeNotification o : objs) {
- Objects.requireNonNull(o, "Only non null objects can be notified");
- immediate.add(o);
- }
- }
-
- /**
- * Add any object which should be notified when safemode is ended and after
- * the configured safemode delay.
- * @param objs List of objects to be notified.
- */
- public void notifyAfterDelay(SafeModeNotification...objs) {
- for (SafeModeNotification o : objs) {
- Objects.requireNonNull(o, "Only non null objects can be notified");
- delayed.add(o);
- }
- }
-
- /**
- * Set SafeMode status based on
- * {@link org.apache.hadoop.hdds.scm.events.SCMEvents#SAFE_MODE_STATUS}.
- *
- * Inform BlockManager, ScmClientProtocolServer, ScmPipeline Manager and
- * Replication Manager status about safeMode status.
- *
- * @param safeModeStatus
- * @param publisher
- */
- @Override
- public void onMessage(SafeModeStatus safeModeStatus,
- EventPublisher publisher) {
- isInSafeMode.set(safeModeStatus.getSafeModeStatus());
- for (SafeModeNotification s : immediate) {
- s.handleSafeModeTransition(safeModeStatus);
- }
- if (!isInSafeMode.get()) {
- final Thread safeModeExitThread = new Thread(() -> {
- try {
- Thread.sleep(waitTime);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- for (SafeModeNotification s : delayed) {
- s.handleSafeModeTransition(safeModeStatus);
- }
- });
-
- safeModeExitThread.setDaemon(true);
- safeModeExitThread.start();
- }
-
- }
-
- public boolean getSafeModeStatus() {
- return isInSafeMode.get();
- }
-}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeNotification.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeNotification.java
deleted file mode 100644
index c0e6810..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeNotification.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * <p>http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * <p>Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.safemode;
-
-/**
- * Interface which should be implemented by any object that wishes to be
- * notified by the SafeModeManager when the safe mode state changes.
- */
-public interface SafeModeNotification {
- void handleSafeModeTransition(SCMSafeModeManager.SafeModeStatus status);
-}
\ No newline at end of file
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModePrecheck.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModePrecheck.java
index b63d04e..6a0001c 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModePrecheck.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModePrecheck.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.hdds.scm.safemode;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmOps;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
@@ -33,7 +33,7 @@
private AtomicBoolean inSafeMode;
public static final String PRECHECK_TYPE = "SafeModePrecheck";
- public SafeModePrecheck(Configuration conf) {
+ public SafeModePrecheck(ConfigurationSource conf) {
boolean safeModeEnabled = conf.getBoolean(
HddsConfigKeys.HDDS_SCM_SAFEMODE_ENABLED,
HddsConfigKeys.HDDS_SCM_SAFEMODE_ENABLED_DEFAULT);
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
index b88b54f..99f873f 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
@@ -61,6 +61,7 @@
import com.google.common.collect.Maps;
import com.google.protobuf.BlockingService;
+import com.google.protobuf.ProtocolMessageEnum;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_DEFAULT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_KEY;
@@ -86,7 +87,7 @@
private final OzoneConfiguration conf;
private final RPC.Server blockRpcServer;
private final InetSocketAddress blockRpcAddress;
- private final ProtocolMessageMetrics
+ private final ProtocolMessageMetrics<ProtocolMessageEnum>
protocolMessageMetrics;
/**
@@ -104,7 +105,8 @@
ProtobufRpcEngine.class);
protocolMessageMetrics =
- ProtocolMessageMetrics.create("ScmBlockLocationProtocol",
+ ProtocolMessageMetrics.create(
+ "ScmBlockLocationProtocol",
"SCM Block location protocol counters",
ScmBlockLocationProtocolProtos.Type.values());
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index eb5e2c0..2eea4f6 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -21,37 +21,45 @@
*/
package org.apache.hadoop.hdds.scm.server;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
-import com.google.protobuf.BlockingService;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmOps;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerLocationProtocolProtos;
-import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager;
-import org.apache.hadoop.hdds.scm.safemode.SafeModeNotification;
-import org.apache.hadoop.hdds.utils.HddsServerUtil;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.ScmUtils;
-import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException;
-import org.apache.hadoop.hdds.scm.safemode.SafeModePrecheck;
import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
import org.apache.hadoop.hdds.scm.container.ContainerReplica;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
-import org.apache.hadoop.hdds.scm.container.ContainerInfo;
-import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
+import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocolServerSideTranslatorPB;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolPB;
+import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager.SafeModeStatus;
+import org.apache.hadoop.hdds.scm.safemode.SafeModePrecheck;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.hdds.utils.HddsServerUtil;
+import org.apache.hadoop.hdds.utils.ProtocolMessageMetrics;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
@@ -63,41 +71,28 @@
import org.apache.hadoop.ozone.audit.AuditMessage;
import org.apache.hadoop.ozone.audit.Auditor;
import org.apache.hadoop.ozone.audit.SCMAction;
-import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocolServerSideTranslatorPB;
-import org.apache.hadoop.hdds.utils.ProtocolMessageMetrics;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.stream.Collectors;
-
-import static org.apache.hadoop.hdds.protocol.proto
- .StorageContainerLocationProtocolProtos
- .StorageContainerLocationProtocolService.newReflectiveBlockingService;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_CLIENT_ADDRESS_KEY;
-
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_HANDLER_COUNT_DEFAULT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_HANDLER_COUNT_KEY;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+import com.google.protobuf.BlockingService;
+import com.google.protobuf.ProtocolMessageEnum;
+import static org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StorageContainerLocationProtocolService.newReflectiveBlockingService;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_KEY;
+import static org.apache.hadoop.hdds.scm.server.StorageContainerManager.startRpcServer;
import static org.apache.hadoop.hdds.server.ServerUtils.getRemoteUserName;
import static org.apache.hadoop.hdds.server.ServerUtils.updateRPCListenAddress;
-import static org.apache.hadoop.hdds.scm.server.StorageContainerManager
- .startRpcServer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* The RPC server that listens to requests from clients.
*/
public class SCMClientProtocolServer implements
- StorageContainerLocationProtocol, Auditor, SafeModeNotification {
+ StorageContainerLocationProtocol, Auditor,
+ EventHandler<SafeModeStatus> {
private static final Logger LOG =
LoggerFactory.getLogger(SCMClientProtocolServer.class);
private static final AuditLogger AUDIT =
@@ -107,7 +102,7 @@
private final StorageContainerManager scm;
private final OzoneConfiguration conf;
private SafeModePrecheck safeModePrecheck;
- private final ProtocolMessageMetrics protocolMetrics;
+ private final ProtocolMessageMetrics<ProtocolMessageEnum> protocolMetrics;
public SCMClientProtocolServer(OzoneConfiguration conf,
StorageContainerManager scm) throws IOException {
@@ -226,57 +221,93 @@
}
+ private ContainerWithPipeline getContainerWithPipelineCommon(
+ long containerID) throws IOException {
+ final ContainerID cid = ContainerID.valueof(containerID);
+ final ContainerInfo container = scm.getContainerManager()
+ .getContainer(cid);
+
+ if (safeModePrecheck.isInSafeMode()) {
+ if (container.isOpen()) {
+ if (!hasRequiredReplicas(container)) {
+ throw new SCMException("Open container " + containerID + " doesn't"
+ + " have enough replicas to service this operation in "
+ + "Safe mode.", ResultCodes.SAFE_MODE_EXCEPTION);
+ }
+ }
+ }
+
+ Pipeline pipeline;
+ try {
+ pipeline = container.isOpen() ? scm.getPipelineManager()
+ .getPipeline(container.getPipelineID()) : null;
+ } catch (PipelineNotFoundException ex) {
+ // The pipeline is destroyed.
+ pipeline = null;
+ }
+
+ if (pipeline == null) {
+ pipeline = scm.getPipelineManager().createPipeline(
+ HddsProtos.ReplicationType.STAND_ALONE,
+ container.getReplicationFactor(),
+ scm.getContainerManager()
+ .getContainerReplicas(cid).stream()
+ .map(ContainerReplica::getDatanodeDetails)
+ .collect(Collectors.toList()));
+ }
+
+ return new ContainerWithPipeline(container, pipeline);
+ }
+
@Override
public ContainerWithPipeline getContainerWithPipeline(long containerID)
throws IOException {
- final ContainerID cid = ContainerID.valueof(containerID);
+ getScm().checkAdminAccess(null);
+
try {
- final ContainerInfo container = scm.getContainerManager()
- .getContainer(cid);
-
- if (safeModePrecheck.isInSafeMode()) {
- if (container.isOpen()) {
- if (!hasRequiredReplicas(container)) {
- throw new SCMException("Open container " + containerID + " doesn't"
- + " have enough replicas to service this operation in "
- + "Safe mode.", ResultCodes.SAFE_MODE_EXCEPTION);
- }
- }
- }
- getScm().checkAdminAccess(null);
-
- Pipeline pipeline;
- try {
- pipeline = container.isOpen() ? scm.getPipelineManager()
- .getPipeline(container.getPipelineID()) : null;
- } catch (PipelineNotFoundException ex) {
- // The pipeline is destroyed.
- pipeline = null;
- }
-
- if (pipeline == null) {
- pipeline = scm.getPipelineManager().createPipeline(
- HddsProtos.ReplicationType.STAND_ALONE,
- container.getReplicationFactor(),
- scm.getContainerManager()
- .getContainerReplicas(cid).stream()
- .map(ContainerReplica::getDatanodeDetails)
- .collect(Collectors.toList()));
- }
-
+ ContainerWithPipeline cp = getContainerWithPipelineCommon(containerID);
AUDIT.logReadSuccess(buildAuditMessageForSuccess(
SCMAction.GET_CONTAINER_WITH_PIPELINE,
- Collections.singletonMap("containerID", cid.toString())));
-
- return new ContainerWithPipeline(container, pipeline);
+ Collections.singletonMap("containerID",
+ ContainerID.valueof(containerID).toString())));
+ return cp;
} catch (IOException ex) {
AUDIT.logReadFailure(buildAuditMessageForFailure(
SCMAction.GET_CONTAINER_WITH_PIPELINE,
- Collections.singletonMap("containerID", cid.toString()), ex));
+ Collections.singletonMap("containerID",
+ ContainerID.valueof(containerID).toString()), ex));
throw ex;
}
}
+ @Override
+ public List<ContainerWithPipeline> getContainerWithPipelineBatch(
+ List<Long> containerIDs) throws IOException {
+ getScm().checkAdminAccess(null);
+
+ List<ContainerWithPipeline> cpList = new ArrayList<>();
+
+ for (Long containerID : containerIDs) {
+ try {
+ ContainerWithPipeline cp = getContainerWithPipelineCommon(containerID);
+ cpList.add(cp);
+ } catch (IOException ex) {
+ AUDIT.logReadFailure(buildAuditMessageForFailure(
+ SCMAction.GET_CONTAINER_WITH_PIPELINE_BATCH,
+ Collections.singletonMap("containerID",
+ ContainerID.valueof(containerID).toString()), ex));
+ throw ex;
+ }
+ }
+
+ AUDIT.logReadSuccess(buildAuditMessageForSuccess(
+ SCMAction.GET_CONTAINER_WITH_PIPELINE_BATCH,
+ Collections.singletonMap("containerIDs",
+ containerIDs.stream().map(id -> ContainerID.valueof(id).toString())
+ .collect(Collectors.joining(",")))));
+
+ return cpList;
+ }
/**
* Check if container reported replicas are equal or greater than required
* replication factor.
@@ -602,8 +633,8 @@
}
@Override
- public void handleSafeModeTransition(
- SCMSafeModeManager.SafeModeStatus status) {
- safeModePrecheck.setInSafeMode(status.getSafeModeStatus());
+ public void onMessage(SafeModeStatus status,
+ EventPublisher publisher) {
+ safeModePrecheck.setInSafeMode(status.isInSafeMode());
}
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
index fd38010..ad7f65a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
@@ -43,11 +43,12 @@
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMRegisteredResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto;
-import org.apache.hadoop.hdds.utils.HddsServerUtil;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineReportFromDatanode;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.ReportFromDatanode;
import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.hdds.utils.HddsServerUtil;
+import org.apache.hadoop.hdds.utils.ProtocolMessageMetrics;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
@@ -68,25 +69,22 @@
import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand;
import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-import org.apache.hadoop.hdds.utils.ProtocolMessageMetrics;
import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolPB;
import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolServerSideTranslatorPB;
+import org.apache.hadoop.security.authorize.PolicyProvider;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import com.google.protobuf.BlockingService;
+import com.google.protobuf.ProtocolMessageEnum;
import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.closeContainerCommand;
+import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.closePipelineCommand;
+import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.createPipelineCommand;
import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.deleteBlocksCommand;
import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.deleteContainerCommand;
import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.replicateContainerCommand;
import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.reregisterCommand;
-import static org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type
- .createPipelineCommand;
-import static org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type
- .closePipelineCommand;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_DEFAULT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_KEY;
@@ -95,8 +93,6 @@
import static org.apache.hadoop.hdds.scm.server.StorageContainerManager.startRpcServer;
import static org.apache.hadoop.hdds.server.ServerUtils.getRemoteUserName;
import static org.apache.hadoop.hdds.server.ServerUtils.updateRPCListenAddress;
-
-import org.apache.hadoop.security.authorize.PolicyProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -121,7 +117,7 @@
private final InetSocketAddress datanodeRpcAddress;
private final SCMDatanodeHeartbeatDispatcher heartbeatDispatcher;
private final EventPublisher eventPublisher;
- private ProtocolMessageMetrics protocolMessageMetrics;
+ private ProtocolMessageMetrics<ProtocolMessageEnum> protocolMessageMetrics;
public SCMDatanodeProtocolServer(final OzoneConfiguration conf,
OzoneStorageContainerManager scm,
@@ -409,7 +405,8 @@
* Get the ProtocolMessageMetrics for this server.
* @return ProtocolMessageMetrics
*/
- protected ProtocolMessageMetrics getProtocolMessageMetrics() {
+ protected ProtocolMessageMetrics<ProtocolMessageEnum>
+ getProtocolMessageMetrics() {
return ProtocolMessageMetrics
.create("SCMDatanodeProtocol", "SCM Datanode protocol",
StorageContainerDatanodeProtocolProtos.Type.values());
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMHTTPServerConfig.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMHTTPServerConfig.java
index 18eb8b3..2b6fa03 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMHTTPServerConfig.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMHTTPServerConfig.java
@@ -29,7 +29,7 @@
/**
* SCM HTTP Server configuration in Java style configuration class.
*/
-@ConfigGroup(prefix = "hdds.scm.http")
+@ConfigGroup(prefix = "hdds.scm.http.auth")
public class SCMHTTPServerConfig {
@Config(key = "kerberos.principal",
@@ -75,9 +75,17 @@
* ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
*/
public static class ConfigStrings {
+ public static final String HDDS_SCM_HTTP_AUTH_CONFIG_PREFIX =
+ SCMHTTPServerConfig.class.getAnnotation(ConfigGroup.class).prefix() +
+ ".";
+
+ public static final String HDDS_SCM_HTTP_AUTH_TYPE =
+ HDDS_SCM_HTTP_AUTH_CONFIG_PREFIX + "type";
+
public static final String HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY =
- "hdds.scm.http.kerberos.principal";
+ HDDS_SCM_HTTP_AUTH_CONFIG_PREFIX + "kerberos.principal";
+
public static final String HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY =
- "hdds.scm.http.kerberos.keytab";
+ HDDS_SCM_HTTP_AUTH_CONFIG_PREFIX + "kerberos.keytab";
}
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 9dcb8f2..a6cd5d4 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -21,36 +21,33 @@
*/
package org.apache.hadoop.hdds.scm.server;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.RemovalListener;
-import com.google.protobuf.BlockingService;
-
+import javax.management.ObjectName;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Objects;
-import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsUtils;
+import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
-import org.apache.hadoop.hdds.utils.HddsServerUtil;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.ScmConfig;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.block.BlockManager;
import org.apache.hadoop.hdds.scm.block.BlockManagerImpl;
import org.apache.hadoop.hdds.scm.block.DeletedBlockLogImpl;
import org.apache.hadoop.hdds.scm.block.PendingDeleteHandler;
-import org.apache.hadoop.hdds.scm.container.ReplicationManager.ReplicationManagerConfiguration;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicyFactory;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms
- .SCMContainerPlacementMetrics;
-import org.apache.hadoop.hdds.scm.net.NetworkTopology;
-import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl;
-import org.apache.hadoop.hdds.scm.safemode.SafeModeHandler;
-import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager;
import org.apache.hadoop.hdds.scm.command.CommandStatusReportHandler;
import org.apache.hadoop.hdds.scm.container.CloseContainerEventHandler;
import org.apache.hadoop.hdds.scm.container.ContainerActionsHandler;
@@ -59,27 +56,32 @@
import org.apache.hadoop.hdds.scm.container.ContainerManager;
import org.apache.hadoop.hdds.scm.container.ContainerReportHandler;
import org.apache.hadoop.hdds.scm.container.IncrementalContainerReportHandler;
+import org.apache.hadoop.hdds.scm.container.ReplicationManager;
+import org.apache.hadoop.hdds.scm.container.ReplicationManager.ReplicationManagerConfiguration;
import org.apache.hadoop.hdds.scm.container.SCMContainerManager;
-import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicyFactory;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementMetrics;
import org.apache.hadoop.hdds.scm.container.placement.metrics.ContainerStat;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMMetrics;
-import org.apache.hadoop.hdds.scm.container.ReplicationManager;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes;
import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore;
import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreRDBImpl;
+import org.apache.hadoop.hdds.scm.net.NetworkTopology;
+import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl;
import org.apache.hadoop.hdds.scm.node.DeadNodeHandler;
import org.apache.hadoop.hdds.scm.node.NewNodeHandler;
-import org.apache.hadoop.hdds.scm.node.NonHealthyToHealthyNodeHandler;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.node.NodeReportHandler;
+import org.apache.hadoop.hdds.scm.node.NonHealthyToHealthyNodeHandler;
import org.apache.hadoop.hdds.scm.node.SCMNodeManager;
import org.apache.hadoop.hdds.scm.node.StaleNodeHandler;
import org.apache.hadoop.hdds.scm.pipeline.PipelineActionHandler;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.scm.pipeline.PipelineReportHandler;
import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
+import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager;
import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateServer;
@@ -88,6 +90,9 @@
import org.apache.hadoop.hdds.server.ServiceRuntimeInfoImpl;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.hdds.utils.HddsServerUtil;
+import org.apache.hadoop.hdds.utils.HddsVersionInfo;
+import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.RPC;
@@ -104,23 +109,17 @@
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.util.JvmPauseMonitor;
-import org.apache.hadoop.hdds.utils.HddsVersionInfo;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.RemovalListener;
+import com.google.protobuf.BlockingService;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT_DEFAULT;
import org.apache.ratis.grpc.GrpcTlsConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.management.ObjectName;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT_DEFAULT;
-
/**
* StorageContainerManager is the main entry point for the service that
* provides information about
@@ -192,7 +191,6 @@
private JvmPauseMonitor jvmPauseMonitor;
private final OzoneConfiguration configuration;
- private final SafeModeHandler safeModeHandler;
private SCMContainerMetrics scmContainerMetrics;
private SCMContainerPlacementMetrics placementMetrics;
private MetricsSystem ms;
@@ -303,7 +301,7 @@
new PendingDeleteHandler(scmBlockManager.getSCMBlockDeletingService());
ContainerReportHandler containerReportHandler =
- new ContainerReportHandler(scmNodeManager, containerManager);
+ new ContainerReportHandler(scmNodeManager, containerManager, conf);
IncrementalContainerReportHandler incrementalContainerReportHandler =
new IncrementalContainerReportHandler(
@@ -333,10 +331,6 @@
clientProtocolServer = new SCMClientProtocolServer(conf, this);
httpServer = new StorageContainerManagerHttpServer(conf);
- safeModeHandler = new SafeModeHandler(configuration);
- safeModeHandler.notifyImmediately(clientProtocolServer, scmBlockManager);
- safeModeHandler.notifyAfterDelay(replicationManager, pipelineManager);
-
eventQueue.addHandler(SCMEvents.DATANODE_COMMAND, scmNodeManager);
eventQueue.addHandler(SCMEvents.RETRIABLE_DATANODE_COMMAND, scmNodeManager);
eventQueue.addHandler(SCMEvents.NODE_REPORT, nodeReportHandler);
@@ -357,7 +351,13 @@
(DeletedBlockLogImpl) scmBlockManager.getDeletedBlockLog());
eventQueue.addHandler(SCMEvents.PIPELINE_ACTIONS, pipelineActionHandler);
eventQueue.addHandler(SCMEvents.PIPELINE_REPORT, pipelineReportHandler);
- eventQueue.addHandler(SCMEvents.SAFE_MODE_STATUS, safeModeHandler);
+ eventQueue.addHandler(SCMEvents.SAFE_MODE_STATUS, clientProtocolServer);
+ eventQueue.addHandler(SCMEvents.SAFE_MODE_STATUS, scmBlockManager);
+ eventQueue
+ .addHandler(SCMEvents.DELAYED_SAFE_MODE_STATUS, replicationManager);
+ eventQueue
+ .addHandler(SCMEvents.DELAYED_SAFE_MODE_STATUS, pipelineManager);
+
// Emit initial safe mode status, as now handlers are registered.
scmSafeModeManager.emitSafeModeStatus();
@@ -406,13 +406,19 @@
pipelineManager = configurator.getPipelineManager();
} else {
pipelineManager =
- new SCMPipelineManager(conf, scmNodeManager, eventQueue);
+ new SCMPipelineManager(conf, scmNodeManager,
+ scmMetadataStore.getPipelineTable(),
+ eventQueue);
}
if (configurator.getContainerManager() != null) {
containerManager = configurator.getContainerManager();
} else {
- containerManager = new SCMContainerManager(conf, pipelineManager);
+ containerManager =
+ new SCMContainerManager(conf,
+ scmMetadataStore.getContainerTable(),
+ scmMetadataStore.getBatchHandler(),
+ pipelineManager);
}
if (configurator.getScmBlockManager() != null) {
@@ -505,7 +511,7 @@
*
* @param conf
*/
- private void loginAsSCMUser(Configuration conf)
+ private void loginAsSCMUser(ConfigurationSource conf)
throws IOException, AuthenticationException {
if (LOG.isDebugEnabled()) {
ScmConfig scmConfig = configuration.getObject(ScmConfig.class);
@@ -515,18 +521,20 @@
scmConfig.getKerberosKeytab());
}
- if (SecurityUtil.getAuthenticationMethod(conf).equals(
+ Configuration hadoopConf =
+ LegacyHadoopConfigurationSource.asHadoopConfiguration(conf);
+ if (SecurityUtil.getAuthenticationMethod(hadoopConf).equals(
AuthenticationMethod.KERBEROS)) {
- UserGroupInformation.setConfiguration(conf);
+ UserGroupInformation.setConfiguration(hadoopConf);
InetSocketAddress socAddr = HddsServerUtil
.getScmBlockClientBindAddress(conf);
- SecurityUtil.login(conf,
+ SecurityUtil.login(hadoopConf,
ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY,
ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_PRINCIPAL_KEY,
socAddr.getHostName());
} else {
throw new AuthenticationException(SecurityUtil.getAuthenticationMethod(
- conf) + " authentication method not support. "
+ hadoopConf) + " authentication method not support. "
+ "SCM user login failed.");
}
LOG.info("SCM login successful.");
@@ -971,11 +979,6 @@
}
@VisibleForTesting
- public SafeModeHandler getSafeModeHandler() {
- return safeModeHandler;
- }
-
- @VisibleForTesting
public SCMSafeModeManager getScmSafeModeManager() {
return scmSafeModeManager;
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
index 3f963fd..b644978 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
@@ -17,18 +17,18 @@
package org.apache.hadoop.hdds.scm.server;
-import org.apache.hadoop.conf.Configuration;
+import java.io.IOException;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.server.http.BaseHttpServer;
-import java.io.IOException;
-
/**
* HttpServer2 wrapper for the Ozone Storage Container Manager.
*/
public class StorageContainerManagerHttpServer extends BaseHttpServer {
- public StorageContainerManagerHttpServer(Configuration conf)
+ public StorageContainerManagerHttpServer(ConfigurationSource conf)
throws IOException {
super(conf, "scm");
}
@@ -75,4 +75,14 @@
return ScmConfigKeys.OZONE_SCM_HTTP_ENABLED_KEY;
}
+ @Override
+ protected String getHttpAuthType() {
+ return SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_AUTH_TYPE;
+ }
+
+ @Override
+ protected String getHttpAuthConfigPrefix() {
+ return SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_AUTH_CONFIG_PREFIX;
+ }
+
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestHddsServerUtil.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestHddsServerUtil.java
index 843586c..b3cb668 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestHddsServerUtil.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestHddsServerUtil.java
@@ -19,20 +19,18 @@
package org.apache.hadoop.hdds.scm;
-import org.apache.hadoop.conf.Configuration;
+import java.net.InetSocketAddress;
+
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.utils.HddsServerUtil;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.rules.Timeout;
-import java.net.InetSocketAddress;
-
-import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.assertThat;
-
/**
* Test the HDDS server side utilities.
*/
@@ -50,7 +48,7 @@
*/
@Test
public void testMissingScmDataNodeAddress() {
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
thrown.expect(IllegalArgumentException.class);
HddsServerUtil.getScmAddressForDataNodes(conf);
}
@@ -62,7 +60,7 @@
*/
@Test
public void testGetScmDataNodeAddress() {
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
// First try a client address with just a host name. Verify it falls
// back to the default port.
@@ -110,7 +108,7 @@
*/
@Test
public void testScmClientBindHostDefault() {
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
// The bind host should be 0.0.0.0 unless OZONE_SCM_CLIENT_BIND_HOST_KEY
// is set differently.
@@ -156,7 +154,7 @@
*/
@Test
public void testScmDataNodeBindHostDefault() {
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
// The bind host should be 0.0.0.0 unless OZONE_SCM_DATANODE_BIND_HOST_KEY
// is set differently.
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestHddsServerUtils.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestHddsServerUtils.java
index d4cc5a4..56d265a 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestHddsServerUtils.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestHddsServerUtils.java
@@ -18,7 +18,10 @@
package org.apache.hadoop.hdds.scm;
-import org.apache.hadoop.conf.Configuration;
+import java.io.File;
+import java.net.InetSocketAddress;
+import java.util.concurrent.TimeUnit;
+
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.server.ServerUtils;
@@ -26,6 +29,12 @@
import org.apache.hadoop.test.PathUtils;
import org.apache.commons.io.FileUtils;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import org.junit.Rule;
import org.junit.Test;
@@ -34,17 +43,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.net.InetSocketAddress;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
-import static org.junit.Assert.assertEquals;
-
/**
* Unit tests for {@link HddsServerUtil}.
*/
@@ -65,7 +63,7 @@
@SuppressWarnings("StringSplitter")
public void testGetDatanodeAddressWithPort() {
final String scmHost = "host123:100";
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
conf.set(OZONE_SCM_DATANODE_ADDRESS_KEY, scmHost);
final InetSocketAddress address =
HddsServerUtil.getScmAddressForDataNodes(conf);
@@ -79,7 +77,7 @@
@Test
public void testGetDatanodeAddressWithoutPort() {
final String scmHost = "host123";
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
conf.set(OZONE_SCM_DATANODE_ADDRESS_KEY, scmHost);
final InetSocketAddress address =
HddsServerUtil.getScmAddressForDataNodes(conf);
@@ -94,7 +92,7 @@
@Test
public void testDatanodeAddressFallbackToClientNoPort() {
final String scmHost = "host123";
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, scmHost);
final InetSocketAddress address =
HddsServerUtil.getScmAddressForDataNodes(conf);
@@ -111,7 +109,7 @@
@SuppressWarnings("StringSplitter")
public void testDatanodeAddressFallbackToClientWithPort() {
final String scmHost = "host123:100";
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, scmHost);
final InetSocketAddress address =
HddsServerUtil.getScmAddressForDataNodes(conf);
@@ -126,7 +124,7 @@
@Test
public void testDatanodeAddressFallbackToScmNamesNoPort() {
final String scmHost = "host123";
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
conf.set(OZONE_SCM_NAMES, scmHost);
final InetSocketAddress address =
HddsServerUtil.getScmAddressForDataNodes(conf);
@@ -143,7 +141,7 @@
@SuppressWarnings("StringSplitter")
public void testDatanodeAddressFallbackToScmNamesWithPort() {
final String scmHost = "host123:100";
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
conf.set(OZONE_SCM_NAMES, scmHost);
final InetSocketAddress address =
HddsServerUtil.getScmAddressForDataNodes(conf);
@@ -158,7 +156,7 @@
@Test
public void testClientFailsWithMultipleScmNames() {
final String scmHost = "host123,host456";
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
conf.set(OZONE_SCM_NAMES, scmHost);
thrown.expect(IllegalArgumentException.class);
HddsServerUtil.getScmAddressForDataNodes(conf);
@@ -172,7 +170,7 @@
final File testDir = PathUtils.getTestDir(TestHddsServerUtils.class);
final File dbDir = new File(testDir, "scmDbDir");
final File metaDir = new File(testDir, "metaDir"); // should be ignored.
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
conf.set(ScmConfigKeys.OZONE_SCM_DB_DIRS, dbDir.getPath());
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.getPath());
@@ -192,7 +190,7 @@
public void testGetScmDbDirWithFallback() {
final File testDir = PathUtils.getTestDir(TestHddsServerUtils.class);
final File metaDir = new File(testDir, "metaDir");
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.getPath());
try {
assertEquals(metaDir, ServerUtils.getScmDbDir(conf));
@@ -210,7 +208,7 @@
@Test
public void testGetStaleNodeInterval() {
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
// Reset OZONE_SCM_STALENODE_INTERVAL to 300s that
// larger than max limit value.
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHttpServer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHttpServer.java
index 0376864..c21788a 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHttpServer.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHttpServer.java
@@ -26,8 +26,8 @@
import java.util.Arrays;
import java.util.Collection;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.server.StorageContainerManagerHttpServer;
import org.apache.hadoop.hdfs.web.URLConnectionFactory;
import org.apache.hadoop.http.HttpConfig;
@@ -54,7 +54,7 @@
.getTempPath(TestStorageContainerManagerHttpServer.class.getSimpleName());
private static String keystoresDir;
private static String sslConfDir;
- private static Configuration conf;
+ private static OzoneConfiguration conf;
private static URLConnectionFactory connectionFactory;
@Parameters public static Collection<Object[]> policy() {
@@ -76,7 +76,7 @@
File base = new File(BASEDIR);
FileUtil.fullyDelete(base);
base.mkdirs();
- conf = new Configuration();
+ conf = new OzoneConfiguration();
keystoresDir = new File(BASEDIR).getAbsolutePath();
sslConfDir = KeyStoreTestUtil.getClasspathDir(
TestStorageContainerManagerHttpServer.class);
@@ -90,6 +90,7 @@
}
@AfterClass public static void tearDown() throws Exception {
+ connectionFactory.destroy();
FileUtil.fullyDelete(new File(BASEDIR));
KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir);
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
index e046d1c..955efd3 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
@@ -29,8 +29,7 @@
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMCommandProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.container.CloseContainerEventHandler;
@@ -40,9 +39,11 @@
import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore;
+import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreRDBImpl;
+import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider;
-import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider;
import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager;
import org.apache.hadoop.hdds.scm.server.SCMConfigurator;
@@ -55,6 +56,9 @@
import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
import org.apache.hadoop.ozone.protocol.commands.CreatePipelineCommand;
import org.apache.hadoop.test.GenericTestUtils;
+
+import static org.apache.hadoop.ozone.OzoneConsts.GB;
+import static org.apache.hadoop.ozone.OzoneConsts.MB;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -63,10 +67,6 @@
import org.junit.rules.ExpectedException;
import org.junit.rules.TemporaryFolder;
-import static org.apache.hadoop.ozone.OzoneConsts.GB;
-import static org.apache.hadoop.ozone.OzoneConsts.MB;
-
-
/**
* Tests for SCM Block Manager.
*/
@@ -88,6 +88,7 @@
@Rule
public TemporaryFolder folder= new TemporaryFolder();
+ private SCMMetadataStore scmMetadataStore;
@Before
public void setUp() throws Exception {
@@ -105,15 +106,25 @@
// Override the default Node Manager in SCM with this Mock Node Manager.
nodeManager = new MockNodeManager(true, 10);
eventQueue = new EventQueue();
+
+ scmMetadataStore = new SCMMetadataStoreRDBImpl(conf);
+ scmMetadataStore.start(conf);
pipelineManager =
- new SCMPipelineManager(conf, nodeManager, eventQueue);
+ new SCMPipelineManager(conf, nodeManager,
+ scmMetadataStore.getPipelineTable(),
+ eventQueue);
+ pipelineManager.allowPipelineCreation();
+
PipelineProvider mockRatisProvider =
new MockRatisPipelineProvider(nodeManager,
pipelineManager.getStateManager(), conf, eventQueue);
pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS,
mockRatisProvider);
SCMContainerManager containerManager =
- new SCMContainerManager(conf, pipelineManager);
+ new SCMContainerManager(conf,
+ scmMetadataStore.getContainerTable(),
+ scmMetadataStore.getStore(),
+ pipelineManager);
SCMSafeModeManager safeModeManager = new SCMSafeModeManager(conf,
containerManager.getContainers(), pipelineManager, eventQueue) {
@Override
@@ -126,6 +137,7 @@
configurator.setPipelineManager(pipelineManager);
configurator.setContainerManager(containerManager);
configurator.setScmSafeModeManager(safeModeManager);
+ configurator.setMetadataStore(scmMetadataStore);
scm = TestUtils.getScm(conf, configurator);
// Initialize these fields so that the tests can pass.
@@ -139,15 +151,17 @@
factor = HddsProtos.ReplicationFactor.THREE;
type = HddsProtos.ReplicationType.RATIS;
- blockManager.handleSafeModeTransition(
- new SCMSafeModeManager.SafeModeStatus(false));
+
+ blockManager.onMessage(
+ new SCMSafeModeManager.SafeModeStatus(false, false), null);
}
@After
- public void cleanup() {
+ public void cleanup() throws Exception {
scm.stop();
scm.join();
eventQueue.close();
+ scmMetadataStore.stop();
}
@Test
@@ -234,8 +248,8 @@
@Test
public void testAllocateBlockFailureInSafeMode() throws Exception {
- blockManager.handleSafeModeTransition(
- new SCMSafeModeManager.SafeModeStatus(true));
+ blockManager.onMessage(
+ new SCMSafeModeManager.SafeModeStatus(true, true), null);
// Test1: In safe mode expect an SCMException.
thrown.expectMessage("SafeModePrecheck failed for "
+ "allocateBlock");
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java
index 10c38a8..09b41a5 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java
@@ -17,47 +17,51 @@
package org.apache.hadoop.hdds.scm.container;
-import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.conf.Configuration;
+import java.io.File;
+import java.io.IOException;
+
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.TestUtils;
+import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition;
import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider;
import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider;
import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.SCMTestUtils;
import org.apache.hadoop.test.GenericTestUtils;
+
+import org.apache.commons.lang3.RandomUtils;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT;
+import static org.apache.hadoop.hdds.scm.events.SCMEvents.CLOSE_CONTAINER;
+import static org.apache.hadoop.hdds.scm.events.SCMEvents.DATANODE_COMMAND;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
-import java.io.File;
-import java.io.IOException;
-
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE;
-import static org.apache.hadoop.hdds.scm.events.SCMEvents.CLOSE_CONTAINER;
-import static org.apache.hadoop.hdds.scm.events.SCMEvents.DATANODE_COMMAND;
-
/**
* Tests the closeContainerEventHandler class.
*/
public class TestCloseContainerEventHandler {
- private static Configuration configuration;
+ private static OzoneConfiguration configuration;
private static MockNodeManager nodeManager;
private static SCMPipelineManager pipelineManager;
private static SCMContainerManager containerManager;
private static long size;
private static File testDir;
private static EventQueue eventQueue;
+ private static DBStore dbStore;
@BeforeClass
public static void setUp() throws Exception {
@@ -71,14 +75,19 @@
configuration.setInt(ScmConfigKeys.OZONE_SCM_RATIS_PIPELINE_LIMIT, 16);
nodeManager = new MockNodeManager(true, 10);
eventQueue = new EventQueue();
+ dbStore =
+ DBStoreBuilder.createDBStore(configuration, new SCMDBDefinition());
pipelineManager =
- new SCMPipelineManager(configuration, nodeManager, eventQueue);
+ new SCMPipelineManager(configuration, nodeManager,
+ SCMDBDefinition.PIPELINES.getTable(dbStore), eventQueue);
+ pipelineManager.allowPipelineCreation();
PipelineProvider mockRatisProvider =
new MockRatisPipelineProvider(nodeManager,
pipelineManager.getStateManager(), configuration, eventQueue);
pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS,
mockRatisProvider);
- containerManager = new SCMContainerManager(configuration, pipelineManager);
+ containerManager = new SCMContainerManager(configuration,
+ SCMDBDefinition.CONTAINERS.getTable(dbStore), dbStore, pipelineManager);
pipelineManager.triggerPipelineCreation();
eventQueue.addHandler(CLOSE_CONTAINER,
new CloseContainerEventHandler(pipelineManager, containerManager));
@@ -96,6 +105,9 @@
if (pipelineManager != null) {
pipelineManager.close();
}
+ if (dbStore != null) {
+ dbStore.close();
+ }
FileUtil.fullyDelete(testDir);
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
index 41585bc..c7ec835 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
@@ -16,7 +16,7 @@
*/
package org.apache.hadoop.hdds.scm.container;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -59,7 +59,7 @@
@Before
public void setup() throws IOException {
- final Configuration conf = new OzoneConfiguration();
+ final ConfigurationSource conf = new OzoneConfiguration();
this.nodeManager = new MockNodeManager(true, 10);
this.containerManager = Mockito.mock(ContainerManager.class);
this.containerStateManager = new ContainerStateManager(conf);
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestIncrementalContainerReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestIncrementalContainerReportHandler.java
index 7961ba8..3c9e7b6 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestIncrementalContainerReportHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestIncrementalContainerReportHandler.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.hdds.scm.container;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -55,7 +55,7 @@
@Before
public void setup() throws IOException {
- final Configuration conf = new OzoneConfiguration();
+ final ConfigurationSource conf = new OzoneConfiguration();
this.containerManager = Mockito.mock(ContainerManager.class);
this.nodeManager = Mockito.mock(NodeManager.class);
this.containerStateManager = new ContainerStateManager(conf);
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java
index 87d7655..b39a8cd 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.hdds.scm.container;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
@@ -28,6 +28,7 @@
.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import org.apache.hadoop.hdds.scm.container.ReplicationManager.ReplicationManagerConfiguration;
import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementStatusDefault;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.server.events.EventHandler;
@@ -39,6 +40,7 @@
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.ArgumentMatcher;
import org.mockito.Mockito;
import java.io.IOException;
@@ -71,7 +73,7 @@
@Before
public void setup() throws IOException, InterruptedException {
- final Configuration conf = new OzoneConfiguration();
+ final ConfigurationSource conf = new OzoneConfiguration();
final ContainerManager containerManager =
Mockito.mock(ContainerManager.class);
eventQueue = new EventQueue();
@@ -105,6 +107,13 @@
.collect(Collectors.toList());
});
+ Mockito.when(containerPlacementPolicy.validateContainerPlacement(
+ Mockito.anyListOf(DatanodeDetails.class),
+ Mockito.anyInt()
+ )).thenAnswer(invocation -> {
+ return new ContainerPlacementStatusDefault(2, 2, 3);
+ });
+
replicationManager = new ReplicationManager(
new ReplicationManagerConfiguration(),
containerManager,
@@ -591,6 +600,36 @@
Assert.assertEquals(0, datanodeCommandHandler.getInvocation());
}
+ /**
+ * ReplicationManager should close the unhealthy OPEN container.
+ */
+ @Test
+ public void testUnhealthyOpenContainer()
+ throws SCMException, ContainerNotFoundException, InterruptedException {
+ final ContainerInfo container = getContainer(LifeCycleState.OPEN);
+ final ContainerID id = container.containerID();
+ final Set<ContainerReplica> replicas = getReplicas(id, State.OPEN,
+ randomDatanodeDetails(),
+ randomDatanodeDetails());
+ replicas.addAll(getReplicas(id, State.UNHEALTHY, randomDatanodeDetails()));
+
+ containerStateManager.loadContainer(container);
+ for (ContainerReplica replica : replicas) {
+ containerStateManager.updateContainerReplica(id, replica);
+ }
+
+ final CloseContainerEventHandler closeContainerHandler =
+ Mockito.mock(CloseContainerEventHandler.class);
+ eventQueue.addHandler(SCMEvents.CLOSE_CONTAINER, closeContainerHandler);
+
+ replicationManager.processContainersNow();
+
+ // Wait for EventQueue to call the event handler
+ Thread.sleep(100L);
+ Mockito.verify(closeContainerHandler, Mockito.times(1))
+ .onMessage(id, eventQueue);
+ }
+
@Test
public void testGeneratedConfig() {
OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
@@ -605,6 +644,119 @@
}
+ @Test
+ public void additionalReplicaScheduledWhenMisReplicated()
+ throws SCMException, ContainerNotFoundException, InterruptedException {
+ final ContainerInfo container = getContainer(LifeCycleState.CLOSED);
+ final ContainerID id = container.containerID();
+ final UUID originNodeId = UUID.randomUUID();
+ final ContainerReplica replicaOne = getReplicas(
+ id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaTwo = getReplicas(
+ id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaThree = getReplicas(
+ id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+
+ containerStateManager.loadContainer(container);
+ containerStateManager.updateContainerReplica(id, replicaOne);
+ containerStateManager.updateContainerReplica(id, replicaTwo);
+ containerStateManager.updateContainerReplica(id, replicaThree);
+
+ // Ensure a mis-replicated status is returned for any containers in this
+ // test where there are 3 replicas. When there are 2 or 4 replicas
+ // the status returned will be healthy.
+ Mockito.when(containerPlacementPolicy.validateContainerPlacement(
+ Mockito.argThat(new ListOfNElements(3)),
+ Mockito.anyInt()
+ )).thenAnswer(invocation -> {
+ return new ContainerPlacementStatusDefault(1, 2, 3);
+ });
+
+ int currentReplicateCommandCount = datanodeCommandHandler
+ .getInvocationCount(SCMCommandProto.Type.replicateContainerCommand);
+
+ replicationManager.processContainersNow();
+ // Wait for EventQueue to call the event handler
+ Thread.sleep(100L);
+ // At this stage, due to the mocked calls to validteContainerPlacement
+ // the mis-replicated racks will not have improved, so expect to see nothing
+ // scheduled.
+ Assert.assertEquals(currentReplicateCommandCount + 1, datanodeCommandHandler
+ .getInvocationCount(SCMCommandProto.Type.replicateContainerCommand));
+
+ // Now make it so that all containers seem mis-replicated no matter how
+ // many replicas. This will test replicas are not scheduled if the new
+ // replica does not fix the mis-replication.
+ Mockito.when(containerPlacementPolicy.validateContainerPlacement(
+ Mockito.anyList(),
+ Mockito.anyInt()
+ )).thenAnswer(invocation -> {
+ return new ContainerPlacementStatusDefault(1, 2, 3);
+ });
+
+ currentReplicateCommandCount = datanodeCommandHandler
+ .getInvocationCount(SCMCommandProto.Type.replicateContainerCommand);
+
+ replicationManager.processContainersNow();
+ // Wait for EventQueue to call the event handler
+ Thread.sleep(100L);
+ // At this stage, due to the mocked calls to validteContainerPlacement
+ // the mis-replicated racks will not have improved, so expect to see nothing
+ // scheduled.
+ Assert.assertEquals(currentReplicateCommandCount, datanodeCommandHandler
+ .getInvocationCount(SCMCommandProto.Type.replicateContainerCommand));
+ }
+
+ @Test
+ public void overReplicatedButRemovingMakesMisReplicated()
+ throws SCMException, ContainerNotFoundException, InterruptedException {
+ // In this test, the excess replica should not be removed.
+ final ContainerInfo container = getContainer(LifeCycleState.CLOSED);
+ final ContainerID id = container.containerID();
+ final UUID originNodeId = UUID.randomUUID();
+ final ContainerReplica replicaOne = getReplicas(
+ id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaTwo = getReplicas(
+ id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaThree = getReplicas(
+ id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaFour = getReplicas(
+ id, State.CLOSED, 1000L, originNodeId, randomDatanodeDetails());
+ final ContainerReplica replicaFive = getReplicas(
+ id, State.UNHEALTHY, 1000L, originNodeId, randomDatanodeDetails());
+
+ containerStateManager.loadContainer(container);
+ containerStateManager.updateContainerReplica(id, replicaOne);
+ containerStateManager.updateContainerReplica(id, replicaTwo);
+ containerStateManager.updateContainerReplica(id, replicaThree);
+ containerStateManager.updateContainerReplica(id, replicaFour);
+ containerStateManager.updateContainerReplica(id, replicaFive);
+
+ // Ensure a mis-replicated status is returned for any containers in this
+ // test where there are exactly 3 replicas checked.
+ Mockito.when(containerPlacementPolicy.validateContainerPlacement(
+ Mockito.argThat(new ListOfNElements(3)),
+ Mockito.anyInt()
+ )).thenAnswer(
+ invocation -> new ContainerPlacementStatusDefault(1, 2, 3));
+
+ int currentDeleteCommandCount = datanodeCommandHandler
+ .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand);
+
+ replicationManager.processContainersNow();
+ // Wait for EventQueue to call the event handler
+ Thread.sleep(100L);
+ // The unhealthy replica should be removed, but not the other replica
+ // as each time we test with 3 replicas, Mockitor ensures it returns
+ // mis-replicated
+ Assert.assertEquals(currentDeleteCommandCount + 1, datanodeCommandHandler
+ .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand));
+
+ Assert.assertTrue(datanodeCommandHandler.received(
+ SCMCommandProto.Type.deleteContainerCommand,
+ replicaFive.getDatanodeDetails()));
+ }
+
@After
public void teardown() throws IOException {
containerStateManager.close();
@@ -658,4 +810,18 @@
}
}
+ class ListOfNElements extends ArgumentMatcher<List> {
+
+ private int expected;
+
+ ListOfNElements(int expected) {
+ this.expected = expected;
+ }
+
+ @Override
+ public boolean matches(Object argument) {
+ return ((List)argument).size() == expected;
+ }
+ }
+
}
\ No newline at end of file
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java
index 405c582..1821e92 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java
@@ -16,24 +16,44 @@
*/
package org.apache.hadoop.hdds.scm.container;
-import org.apache.hadoop.conf.Configuration;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.XceiverClientManager;
-import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
-import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.XceiverClientManager;
+import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.SCMTestUtils;
import org.apache.hadoop.test.GenericTestUtils;
+
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
@@ -42,24 +62,6 @@
import org.junit.Test;
import org.junit.rules.ExpectedException;
-import java.io.File;
-import java.io.IOException;
-import java.util.Random;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.UUID;
-import java.util.Iterator;
-import java.util.Optional;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
/**
* Tests for Container ContainerManager.
@@ -67,7 +69,7 @@
public class TestSCMContainerManager {
private static SCMContainerManager containerManager;
private static MockNodeManager nodeManager;
- private static PipelineManager pipelineManager;
+ private static SCMPipelineManager pipelineManager;
private static File testDir;
private static XceiverClientManager xceiverClientManager;
private static Random random;
@@ -78,9 +80,10 @@
@Rule
public ExpectedException thrown = ExpectedException.none();
+
@BeforeClass
public static void setUp() throws Exception {
- Configuration conf = SCMTestUtils.getConf();
+ OzoneConfiguration conf = SCMTestUtils.getConf();
testDir = GenericTestUtils
.getTestDir(TestSCMContainerManager.class.getSimpleName());
@@ -95,9 +98,15 @@
throw new IOException("Unable to create test directory path");
}
nodeManager = new MockNodeManager(true, 10);
+ DBStore dbStore = DBStoreBuilder.createDBStore(conf, new SCMDBDefinition());
pipelineManager =
- new SCMPipelineManager(conf, nodeManager, new EventQueue());
- containerManager = new SCMContainerManager(conf, pipelineManager);
+ new SCMPipelineManager(conf, nodeManager,
+ SCMDBDefinition.PIPELINES.getTable(dbStore), new EventQueue());
+ pipelineManager.allowPipelineCreation();
+ containerManager = new SCMContainerManager(conf,
+ SCMDBDefinition.CONTAINERS.getTable(dbStore),
+ dbStore,
+ pipelineManager);
xceiverClientManager = new XceiverClientManager(conf);
replicationFactor = SCMTestUtils.getReplicationFactor(conf);
replicationType = SCMTestUtils.getReplicationType(conf);
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestUnknownContainerReport.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestUnknownContainerReport.java
new file mode 100644
index 0000000..1c2cdd0
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestUnknownContainerReport.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.scm.container;
+
+import static org.apache.hadoop.hdds.scm.TestUtils.getContainer;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.scm.ScmConfig;
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.server
+ .SCMDatanodeHeartbeatDispatcher.ContainerReportFromDatanode;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+/**
+ * Test container deletion behaviour of unknown containers
+ * that reported by Datanodes.
+ */
+public class TestUnknownContainerReport {
+
+ private NodeManager nodeManager;
+ private ContainerManager containerManager;
+ private ContainerStateManager containerStateManager;
+ private EventPublisher publisher;
+
+ @Before
+ public void setup() throws IOException {
+ final ConfigurationSource conf = new OzoneConfiguration();
+ this.nodeManager = new MockNodeManager(true, 10);
+ this.containerManager = Mockito.mock(ContainerManager.class);
+ this.containerStateManager = new ContainerStateManager(conf);
+ this.publisher = Mockito.mock(EventPublisher.class);
+
+ Mockito.when(containerManager.getContainer(Mockito.any(ContainerID.class)))
+ .thenThrow(new ContainerNotFoundException());
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ containerStateManager.close();
+ }
+
+ @Test
+ public void testUnknownContainerNotDeleted() throws IOException {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ sendContainerReport(conf);
+
+ // By default, unknown containers won't be taken delete action by SCM
+ verify(publisher, times(0)).fireEvent(
+ Mockito.eq(SCMEvents.DATANODE_COMMAND),
+ Mockito.any(CommandForDatanode.class));
+ }
+
+ @Test
+ public void testUnknownContainerDeleted() throws IOException {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(
+ ScmConfig.HDDS_SCM_UNKNOWN_CONTAINER_ACTION,
+ ContainerReportHandler.UNKNOWN_CONTAINER_ACTION_DELETE);
+
+ sendContainerReport(conf);
+ verify(publisher, times(1)).fireEvent(
+ Mockito.eq(SCMEvents.DATANODE_COMMAND),
+ Mockito.any(CommandForDatanode.class));
+ }
+
+ /**
+ * Trigger datanode to send unknown container report to SCM.
+ * @param conf OzoneConfiguration instance to initialize
+ * ContainerReportHandler
+ */
+ private void sendContainerReport(OzoneConfiguration conf) {
+ ContainerReportHandler reportHandler = new ContainerReportHandler(
+ nodeManager, containerManager, conf);
+
+ ContainerInfo container = getContainer(LifeCycleState.CLOSED);
+ Iterator<DatanodeDetails> nodeIterator = nodeManager
+ .getNodes(NodeState.HEALTHY).iterator();
+ DatanodeDetails datanode = nodeIterator.next();
+
+ ContainerReportsProto containerReport = getContainerReportsProto(
+ container.containerID(), ContainerReplicaProto.State.CLOSED,
+ datanode.getUuidString());
+ ContainerReportFromDatanode containerReportFromDatanode =
+ new ContainerReportFromDatanode(datanode, containerReport);
+ reportHandler.onMessage(containerReportFromDatanode, publisher);
+ }
+
+ private static ContainerReportsProto getContainerReportsProto(
+ final ContainerID containerId, final ContainerReplicaProto.State state,
+ final String originNodeId) {
+ final ContainerReportsProto.Builder crBuilder =
+ ContainerReportsProto.newBuilder();
+ final ContainerReplicaProto replicaProto =
+ ContainerReplicaProto.newBuilder()
+ .setContainerID(containerId.getId())
+ .setState(state)
+ .setOriginNodeId(originNodeId)
+ .setFinalhash("e16cc9d6024365750ed8dbd194ea46d2")
+ .setSize(5368709120L)
+ .setUsed(2000000000L)
+ .setKeyCount(100000000L)
+ .setReadCount(100000000L)
+ .setWriteCount(100000000L)
+ .setReadBytes(2000000000L)
+ .setWriteBytes(2000000000L)
+ .setBlockCommitSequenceId(10000L)
+ .setDeleteTransactionId(0)
+ .build();
+ return crBuilder.addReports(replicaProto).build();
+ }
+
+}
\ No newline at end of file
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java
index a454de2..842c494 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java
@@ -16,11 +16,15 @@
*/
package org.apache.hadoop.hdds.scm.container.placement.algorithms;
-import org.apache.hadoop.conf.Configuration;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
+import org.apache.hadoop.hdds.scm.ContainerPlacementStatus;
import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
@@ -30,19 +34,15 @@
import org.apache.hadoop.hdds.scm.net.NodeSchema;
import org.apache.hadoop.hdds.scm.net.NodeSchemaManager;
import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
import static org.apache.hadoop.hdds.scm.net.NetConstants.LEAF_SCHEMA;
import static org.apache.hadoop.hdds.scm.net.NetConstants.RACK_SCHEMA;
import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT_SCHEMA;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
import static org.mockito.Matchers.anyObject;
+import org.mockito.Mockito;
import static org.mockito.Mockito.when;
/**
@@ -56,7 +56,7 @@
// node storage capacity
private final long storageCapacity = 100L;
// configuration
- private Configuration conf;
+ private OzoneConfiguration conf;
// node manager
private NodeManager nodeManager;
@@ -133,6 +133,12 @@
int nodesRequired, long sizeRequired) {
return null;
}
+
+ @Override
+ public ContainerPlacementStatus
+ validateContainerPlacement(List<DatanodeDetails> dns, int replicas) {
+ return new ContainerPlacementStatusDefault(1, 1, 1);
+ }
}
@Test(expected = SCMException.class)
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementStatusDefault.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementStatusDefault.java
new file mode 100644
index 0000000..5ef036c
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementStatusDefault.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.container.placement.algorithms;
+
+import org.junit.Test;
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * Test for the ContainerPlacementStatusDefault class.
+ */
+
+public class TestContainerPlacementStatusDefault {
+
+ @Test
+ public void testPlacementSatisfiedCorrectly() {
+ ContainerPlacementStatusDefault stat =
+ new ContainerPlacementStatusDefault(1, 1, 1);
+ assertTrue(stat.isPolicySatisfied());
+ assertEquals(0, stat.misReplicationCount());
+
+ // Requires 2 racks, but cluster only has 1
+ stat = new ContainerPlacementStatusDefault(1, 2, 1);
+ assertTrue(stat.isPolicySatisfied());
+ assertEquals(0, stat.misReplicationCount());
+
+ stat = new ContainerPlacementStatusDefault(2, 2, 3);
+ assertTrue(stat.isPolicySatisfied());
+ assertEquals(0, stat.misReplicationCount());
+
+ stat = new ContainerPlacementStatusDefault(3, 2, 3);
+ assertTrue(stat.isPolicySatisfied());
+ assertEquals(0, stat.misReplicationCount());
+ }
+
+ @Test
+ public void testPlacementNotSatisfied() {
+ ContainerPlacementStatusDefault stat =
+ new ContainerPlacementStatusDefault(1, 2, 2);
+ assertFalse(stat.isPolicySatisfied());
+ assertEquals(1, stat.misReplicationCount());
+
+ // Zero rack, but need 2 - shouldn't really happen in practice
+ stat = new ContainerPlacementStatusDefault(0, 2, 1);
+ assertFalse(stat.isPolicySatisfied());
+ assertEquals(2, stat.misReplicationCount());
+
+ stat = new ContainerPlacementStatusDefault(2, 3, 3);
+ assertFalse(stat.isPolicySatisfied());
+ assertEquals(1, stat.misReplicationCount());
+ }
+
+}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java
index ddca0fa..afefc9a 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java
@@ -21,7 +21,7 @@
import java.util.List;
import java.util.Map;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
@@ -43,7 +43,7 @@
@Test
public void chooseDatanodes() throws SCMException {
//given
- Configuration conf = new OzoneConfiguration();
+ ConfigurationSource conf = new OzoneConfiguration();
List<DatanodeDetails> datanodes = new ArrayList<>();
for (int i = 0; i < 7; i++) {
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java
index 992f1c5..5019ed4 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java
@@ -21,11 +21,12 @@
import java.util.Collection;
import java.util.List;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
+import org.apache.hadoop.hdds.scm.ContainerPlacementStatus;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.net.NetConstants;
@@ -35,7 +36,7 @@
import org.apache.hadoop.hdds.scm.net.NodeSchemaManager;
import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import static org.apache.hadoop.hdds.scm.net.NetConstants.LEAF_SCHEMA;
import static org.apache.hadoop.hdds.scm.net.NetConstants.RACK_SCHEMA;
import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT_SCHEMA;
@@ -43,6 +44,7 @@
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import org.junit.Assert;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.junit.Assume.assumeTrue;
@@ -60,7 +62,7 @@
@RunWith(Parameterized.class)
public class TestSCMContainerPlacementRackAware {
private NetworkTopology cluster;
- private Configuration conf;
+ private ConfigurationSource conf;
private NodeManager nodeManager;
private Integer datanodeCount;
private List<DatanodeDetails> datanodes = new ArrayList<>();
@@ -105,6 +107,8 @@
// create mock node manager
nodeManager = Mockito.mock(NodeManager.class);
+ when(nodeManager.getClusterNetworkTopologyMap())
+ .thenReturn(cluster);
when(nodeManager.getNodes(NodeState.HEALTHY))
.thenReturn(new ArrayList<>(datanodes));
when(nodeManager.getNodeStat(anyObject()))
@@ -377,4 +381,70 @@
Assert.assertTrue(cluster.isSameParent(datanodeDetails.get(1),
datanodeDetails.get(2)));
}
+
+ @Test
+ public void testvalidateContainerPlacement() {
+ // Only run this test for the full set of DNs. 5 DNs per rack on 3 racks.
+ assumeTrue(datanodeCount == 15);
+ List<DatanodeDetails> dns = new ArrayList<>();
+ // First 5 node are on the same rack
+ dns.add(datanodes.get(0));
+ dns.add(datanodes.get(1));
+ dns.add(datanodes.get(2));
+ ContainerPlacementStatus stat = policy.validateContainerPlacement(dns, 3);
+ assertFalse(stat.isPolicySatisfied());
+ assertEquals(1, stat.misReplicationCount());
+
+ // Pick a new list which spans 2 racks
+ dns = new ArrayList<>();
+ dns.add(datanodes.get(0));
+ dns.add(datanodes.get(1));
+ dns.add(datanodes.get(5)); // This is on second rack
+ stat = policy.validateContainerPlacement(dns, 3);
+ assertTrue(stat.isPolicySatisfied());
+ assertEquals(0, stat.misReplicationCount());
+
+ // Pick single DN, expecting 3 replica. Policy is not met.
+ dns = new ArrayList<>();
+ dns.add(datanodes.get(0));
+ stat = policy.validateContainerPlacement(dns, 3);
+ assertFalse(stat.isPolicySatisfied());
+ assertEquals(1, stat.misReplicationCount());
+
+ // Pick single DN, expecting 1 replica. Policy is met.
+ dns = new ArrayList<>();
+ dns.add(datanodes.get(0));
+ stat = policy.validateContainerPlacement(dns, 1);
+ assertTrue(stat.isPolicySatisfied());
+ assertEquals(0, stat.misReplicationCount());
+ }
+
+ @Test
+ public void testvalidateContainerPlacementSingleRackCluster() {
+ assumeTrue(datanodeCount == 5);
+
+ // All nodes are on the same rack in this test, and the cluster only has
+ // one rack.
+ List<DatanodeDetails> dns = new ArrayList<>();
+ dns.add(datanodes.get(0));
+ dns.add(datanodes.get(1));
+ dns.add(datanodes.get(2));
+ ContainerPlacementStatus stat = policy.validateContainerPlacement(dns, 3);
+ assertTrue(stat.isPolicySatisfied());
+ assertEquals(0, stat.misReplicationCount());
+
+ // Single DN - policy met as cluster only has one rack.
+ dns = new ArrayList<>();
+ dns.add(datanodes.get(0));
+ stat = policy.validateContainerPlacement(dns, 3);
+ assertTrue(stat.isPolicySatisfied());
+ assertEquals(0, stat.misReplicationCount());
+
+ // Single DN - only 1 replica expected
+ dns = new ArrayList<>();
+ dns.add(datanodes.get(0));
+ stat = policy.validateContainerPlacement(dns, 1);
+ assertTrue(stat.isPolicySatisfied());
+ assertEquals(0, stat.misReplicationCount());
+ }
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java
index 91509a0..fb8d2e0 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java
@@ -19,17 +19,21 @@
import java.util.ArrayList;
import java.util.List;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
+import org.apache.hadoop.hdds.scm.ContainerPlacementStatus;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.junit.Assert;
import org.junit.Test;
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.assertFalse;
import static org.mockito.Matchers.anyObject;
import org.mockito.Mockito;
import static org.mockito.Mockito.when;
@@ -42,7 +46,7 @@
@Test
public void chooseDatanodes() throws SCMException {
//given
- Configuration conf = new OzoneConfiguration();
+ ConfigurationSource conf = new OzoneConfiguration();
List<DatanodeDetails> datanodes = new ArrayList<>();
for (int i = 0; i < 5; i++) {
@@ -87,4 +91,40 @@
}
}
+
+ @Test
+ public void testPlacementPolicySatisified() {
+ //given
+ ConfigurationSource conf = new OzoneConfiguration();
+
+ List<DatanodeDetails> datanodes = new ArrayList<>();
+ for (int i = 0; i < 3; i++) {
+ datanodes.add(MockDatanodeDetails.randomDatanodeDetails());
+ }
+
+ NodeManager mockNodeManager = Mockito.mock(NodeManager.class);
+ SCMContainerPlacementRandom scmContainerPlacementRandom =
+ new SCMContainerPlacementRandom(mockNodeManager, conf, null, true,
+ null);
+ ContainerPlacementStatus status =
+ scmContainerPlacementRandom.validateContainerPlacement(datanodes, 3);
+ assertTrue(status.isPolicySatisfied());
+ assertEquals(0, status.misReplicationCount());
+
+ status = scmContainerPlacementRandom.validateContainerPlacement(
+ new ArrayList<DatanodeDetails>(), 3);
+ assertFalse(status.isPolicySatisfied());
+
+ // Only expect 1 more replica to give us one rack on this policy.
+ assertEquals(1, status.misReplicationCount(), 3);
+
+ datanodes = new ArrayList<DatanodeDetails>();
+ datanodes.add(MockDatanodeDetails.randomDatanodeDetails());
+ status = scmContainerPlacementRandom.validateContainerPlacement(
+ datanodes, 3);
+ assertTrue(status.isPolicySatisfied());
+
+ // Only expect 1 more replica to give us one rack on this policy.
+ assertEquals(0, status.misReplicationCount(), 3);
+ }
}
\ No newline at end of file
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java
index 8808546..6ce66a2 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java
@@ -18,54 +18,67 @@
package org.apache.hadoop.hdds.scm.node;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.TimeoutException;
+
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.SCMContainerManager;
-import org.apache.hadoop.hdds.scm.PlacementPolicy;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms
- .SCMContainerPlacementCapacity;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.SCMTestUtils;
import org.apache.hadoop.test.PathUtils;
+
+import org.apache.commons.io.IOUtils;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY;
+import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.CONTAINERS;
+import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.PIPELINES;
+import org.junit.After;
+import static org.junit.Assert.assertEquals;
+import org.junit.Before;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.mockito.Mockito;
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.TimeoutException;
-
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_DB_CACHE_SIZE_DEFAULT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_DB_CACHE_SIZE_MB;
-import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState
- .HEALTHY;
-import static org.junit.Assert.assertEquals;
-
/**
* Test for different container placement policy.
*/
public class TestContainerPlacement {
+
@Rule
public ExpectedException thrown = ExpectedException.none();
+ private DBStore dbStore;
+ @Before
+ public void createDbStore() throws IOException {
+ dbStore =
+ DBStoreBuilder.createDBStore(getConf(), new SCMDBDefinition());
+ }
+
+ @After
+ public void destroyDBStore() throws Exception {
+ dbStore.close();
+ }
/**
* Returns a new copy of Configuration.
*
@@ -101,14 +114,16 @@
return nodeManager;
}
- SCMContainerManager createContainerManager(Configuration config,
+ SCMContainerManager createContainerManager(ConfigurationSource config,
NodeManager scmNodeManager) throws IOException {
EventQueue eventQueue = new EventQueue();
- final int cacheSize = config.getInt(OZONE_SCM_DB_CACHE_SIZE_MB,
- OZONE_SCM_DB_CACHE_SIZE_DEFAULT);
+
PipelineManager pipelineManager =
- new SCMPipelineManager(config, scmNodeManager, eventQueue);
- return new SCMContainerManager(config, pipelineManager);
+ new SCMPipelineManager(config, scmNodeManager,
+ PIPELINES.getTable(dbStore), eventQueue);
+ return new SCMContainerManager(config, CONTAINERS.getTable(dbStore),
+ dbStore,
+ pipelineManager);
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
index 3e196a6..6a6d328 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
@@ -26,6 +26,7 @@
import java.util.Arrays;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.apache.hadoop.fs.FileUtil;
@@ -86,6 +87,8 @@
@Before
public void setup() throws IOException, AuthenticationException {
OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setTimeDuration(HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT,
+ 0, TimeUnit.SECONDS);
storageDir = GenericTestUtils.getTempPath(
TestDeadNodeHandler.class.getSimpleName() + UUID.randomUUID());
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir);
@@ -127,6 +130,10 @@
StorageReportProto storageOne = TestUtils.createStorageReport(
datanode1.getUuid(), storagePath, 100, 10, 90, null);
+ // Exit safemode, as otherwise the safemode precheck will prevent pipelines
+ // from getting created. Due to how this test is wired up, safemode will
+ // not exit when the DNs are registered directly with the node manager.
+ scm.exitSafeMode();
// Standalone pipeline now excludes the nodes which are already used,
// is the a proper behavior. Adding 9 datanodes for now to make the
// test case happy.
@@ -152,8 +159,11 @@
nodeManager.register(MockDatanodeDetails.randomDatanodeDetails(),
TestUtils.createNodeReport(storageOne), null);
- LambdaTestUtils.await(120000, 10000,
- () -> pipelineManager.getPipelines(RATIS, THREE).size() == 3);
+ LambdaTestUtils.await(120000, 1000,
+ () -> {
+ pipelineManager.triggerPipelineCreation();
+ return pipelineManager.getPipelines(RATIS, THREE).size() == 3;
+ });
TestUtils.openAllRatisPipelines(pipelineManager);
ContainerInfo container1 =
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
index 91f437d..8b17495 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
@@ -34,6 +34,8 @@
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.NodeReportFromDatanode;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto;
import org.apache.hadoop.hdds.scm.HddsTestUtils;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -42,6 +44,7 @@
import org.apache.hadoop.hdds.scm.net.NetworkTopology;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
@@ -69,12 +72,14 @@
import org.junit.Assert;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
+import org.mockito.Mockito;
/**
* Test the SCM Node Manager class.
@@ -877,8 +882,6 @@
* @throws TimeoutException
*/
@Test
- @Ignore
- // TODO: Enable this after we implement NodeReportEvent handler.
public void testScmNodeReportUpdate()
throws IOException, InterruptedException, TimeoutException,
AuthenticationException {
@@ -896,6 +899,8 @@
try (SCMNodeManager nodeManager = createNodeManager(conf)) {
DatanodeDetails datanodeDetails =
TestUtils.createRandomDatanodeAndRegister(nodeManager);
+ NodeReportHandler nodeReportHandler = new NodeReportHandler(nodeManager);
+ EventPublisher publisher = Mockito.mock(EventPublisher.class);
final long capacity = 2000;
final long usedPerHeartbeat = 100;
UUID dnId = datanodeDetails.getUuid();
@@ -906,7 +911,10 @@
StorageReportProto report = TestUtils
.createStorageReport(dnId, storagePath, capacity, scmUsed,
remaining, null);
-
+ NodeReportProto nodeReportProto = TestUtils.createNodeReport(report);
+ nodeReportHandler.onMessage(
+ new NodeReportFromDatanode(datanodeDetails, nodeReportProto),
+ publisher);
nodeManager.processHeartbeat(datanodeDetails);
Thread.sleep(100);
}
@@ -944,7 +952,7 @@
// Compare the result from
// NodeManager#getNodeStats and NodeManager#getNodeStat
SCMNodeStat stat1 = nodeManager.getNodeStats().
- get(datanodeDetails.getUuid());
+ get(datanodeDetails);
SCMNodeStat stat2 = nodeManager.getNodeStat(datanodeDetails).get();
assertEquals(stat1, stat2);
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java
index 340ebf5..f9fb150 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java
@@ -18,16 +18,16 @@
package org.apache.hadoop.hdds.scm.pipeline;
-import org.apache.hadoop.conf.Configuration;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.EventQueue;
-import java.io.IOException;
-import java.util.List;
-
/**
* Mock Ratis Pipeline Provider for Mock Nodes.
*/
@@ -37,27 +37,27 @@
private boolean isHealthy;
public MockRatisPipelineProvider(NodeManager nodeManager,
- PipelineStateManager stateManager, Configuration conf,
+ PipelineStateManager stateManager, ConfigurationSource conf,
EventPublisher eventPublisher, boolean autoOpen) {
super(nodeManager, stateManager, conf, eventPublisher);
autoOpenPipeline = autoOpen;
}
public MockRatisPipelineProvider(NodeManager nodeManager,
- PipelineStateManager stateManager,
- Configuration conf) {
+ PipelineStateManager stateManager,
+ ConfigurationSource conf) {
super(nodeManager, stateManager, conf, new EventQueue());
}
public MockRatisPipelineProvider(NodeManager nodeManager,
- PipelineStateManager stateManager,
- Configuration conf, boolean isHealthy) {
+ PipelineStateManager stateManager,
+ ConfigurationSource conf, boolean isHealthy) {
super(nodeManager, stateManager, conf, new EventQueue());
this.isHealthy = isHealthy;
}
public MockRatisPipelineProvider(NodeManager nodeManager,
- PipelineStateManager stateManager, Configuration conf,
+ PipelineStateManager stateManager, ConfigurationSource conf,
EventPublisher eventPublisher) {
super(nodeManager, stateManager, conf, eventPublisher);
autoOpenPipeline = true;
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineActionHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineActionHandler.java
new file mode 100644
index 0000000..99443c3
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineActionHandler.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.hdds.scm.pipeline;
+
+import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ClosePipelineInfo;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineAction;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineActionsProto;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineActionsFromDatanode;
+import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.util.UUID;
+
+/**
+ * Test-cases to verify the functionality of PipelineActionHandler.
+ */
+public class TestPipelineActionHandler {
+
+ @Test
+ public void testCloseActionForMissingPipeline()
+ throws PipelineNotFoundException {
+ final PipelineManager manager = Mockito.mock(PipelineManager.class);
+ final EventQueue queue = Mockito.mock(EventQueue.class);
+
+ Mockito.when(manager.getPipeline(Mockito.any(PipelineID.class)))
+ .thenThrow(new PipelineNotFoundException());
+
+ final PipelineActionHandler actionHandler =
+ new PipelineActionHandler(manager, null);
+
+ final PipelineActionsProto actionsProto = PipelineActionsProto.newBuilder()
+ .addPipelineActions(PipelineAction.newBuilder()
+ .setClosePipeline(ClosePipelineInfo.newBuilder()
+ .setPipelineID(HddsProtos.PipelineID.newBuilder()
+ .setId(UUID.randomUUID().toString()).build())
+ .setReason(ClosePipelineInfo.Reason.PIPELINE_FAILED))
+ .setAction(PipelineAction.Action.CLOSE).build())
+ .build();
+ final PipelineActionsFromDatanode pipelineActions =
+ new PipelineActionsFromDatanode(
+ MockDatanodeDetails.randomDatanodeDetails(), actionsProto);
+
+ actionHandler.onMessage(pipelineActions, queue);
+
+ Mockito.verify(queue, Mockito.times(1))
+ .fireEvent(Mockito.any(), Mockito.any(CommandForDatanode.class));
+
+ }
+
+}
\ No newline at end of file
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
index fafc4b0..b153487 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
@@ -18,41 +18,64 @@
package org.apache.hadoop.hdds.scm.pipeline;
-import org.apache.hadoop.conf.Configuration;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.ContainerPlacementStatus;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.MockNodeManager;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.net.*;
+import org.apache.hadoop.hdds.scm.net.NetConstants;
+import org.apache.hadoop.hdds.scm.net.NetworkTopology;
+import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl;
+import org.apache.hadoop.hdds.scm.net.Node;
+import org.apache.hadoop.hdds.scm.net.NodeImpl;
+import org.apache.hadoop.hdds.scm.net.NodeSchema;
+import org.apache.hadoop.hdds.scm.net.NodeSchemaManager;
import org.apache.hadoop.hdds.scm.node.states.Node2PipelineMap;
+
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import java.util.*;
-import java.util.stream.Collectors;
+import java.io.IOException;
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertTrue;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT;
import static org.apache.hadoop.hdds.scm.net.NetConstants.LEAF_SCHEMA;
import static org.apache.hadoop.hdds.scm.net.NetConstants.RACK_SCHEMA;
import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT_SCHEMA;
+import static org.junit.Assert.assertFalse;
/**
* Test for PipelinePlacementPolicy.
*/
public class TestPipelinePlacementPolicy {
private MockNodeManager nodeManager;
+ private PipelineStateManager stateManager;
private OzoneConfiguration conf;
private PipelinePlacementPolicy placementPolicy;
private NetworkTopologyImpl cluster;
private static final int PIPELINE_PLACEMENT_MAX_NODES_COUNT = 10;
+ private static final int PIPELINE_LOAD_LIMIT = 5;
private List<DatanodeDetails> nodesWithOutRackAwareness = new ArrayList<>();
private List<DatanodeDetails> nodesWithRackAwareness = new ArrayList<>();
+ static final Logger LOG =
+ LoggerFactory.getLogger(TestPipelinePlacementPolicy.class);
+
@Before
public void init() throws Exception {
cluster = initTopology();
@@ -60,9 +83,10 @@
nodeManager = new MockNodeManager(cluster, getNodesWithRackAwareness(),
false, PIPELINE_PLACEMENT_MAX_NODES_COUNT);
conf = new OzoneConfiguration();
- conf.setInt(OZONE_DATANODE_PIPELINE_LIMIT, 5);
+ conf.setInt(OZONE_DATANODE_PIPELINE_LIMIT, PIPELINE_LOAD_LIMIT);
+ stateManager = new PipelineStateManager();
placementPolicy = new PipelinePlacementPolicy(
- nodeManager, new PipelineStateManager(), conf);
+ nodeManager, stateManager, conf);
}
private NetworkTopologyImpl initTopology() {
@@ -76,11 +100,14 @@
private List<DatanodeDetails> getNodesWithRackAwareness() {
List<DatanodeDetails> datanodes = new ArrayList<>();
- for (Node node : NODES) {
+ int iter = 0;
+ int delimiter = NODES.length;
+ while (iter < PIPELINE_PLACEMENT_MAX_NODES_COUNT) {
DatanodeDetails datanode = overwriteLocationInNode(
- getNodesWithoutRackAwareness(), node);
+ getNodesWithoutRackAwareness(), NODES[iter % delimiter]);
nodesWithRackAwareness.add(datanode);
datanodes.add(datanode);
+ iter++;
}
return datanodes;
}
@@ -92,7 +119,7 @@
}
@Test
- public void testChooseNodeBasedOnNetworkTopology() throws SCMException {
+ public void testChooseNodeBasedOnNetworkTopology() {
DatanodeDetails anchor = placementPolicy.chooseNode(nodesWithRackAwareness);
// anchor should be removed from healthyNodes after being chosen.
Assert.assertFalse(nodesWithRackAwareness.contains(anchor));
@@ -100,8 +127,11 @@
List<DatanodeDetails> excludedNodes =
new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT);
excludedNodes.add(anchor);
- DatanodeDetails nextNode = placementPolicy.chooseNodeFromNetworkTopology(
- nodeManager.getClusterNetworkTopologyMap(), anchor, excludedNodes);
+ DatanodeDetails nextNode = placementPolicy.chooseNodeBasedOnSameRack(
+ nodesWithRackAwareness, excludedNodes,
+ nodeManager.getClusterNetworkTopologyMap(), anchor);
+ //DatanodeDetails nextNode = placementPolicy.chooseNodeFromNetworkTopology(
+ // nodeManager.getClusterNetworkTopologyMap(), anchor, excludedNodes);
Assert.assertFalse(excludedNodes.contains(nextNode));
// next node should not be the same as anchor.
Assert.assertTrue(anchor.getUuid() != nextNode.getUuid());
@@ -140,6 +170,45 @@
}
@Test
+ public void testPickLowestLoadAnchor() throws IOException{
+ List<DatanodeDetails> healthyNodes = nodeManager
+ .getNodes(HddsProtos.NodeState.HEALTHY);
+
+ int maxPipelineCount = PIPELINE_LOAD_LIMIT * healthyNodes.size()
+ / HddsProtos.ReplicationFactor.THREE.getNumber();
+ for (int i = 0; i < maxPipelineCount; i++) {
+ try {
+ List<DatanodeDetails> nodes = placementPolicy.chooseDatanodes(null,
+ null, HddsProtos.ReplicationFactor.THREE.getNumber(), 0);
+
+ Pipeline pipeline = Pipeline.newBuilder()
+ .setId(PipelineID.randomId())
+ .setState(Pipeline.PipelineState.ALLOCATED)
+ .setType(HddsProtos.ReplicationType.RATIS)
+ .setFactor(HddsProtos.ReplicationFactor.THREE)
+ .setNodes(nodes)
+ .build();
+ nodeManager.addPipeline(pipeline);
+ stateManager.addPipeline(pipeline);
+ } catch (SCMException e) {
+ break;
+ }
+ }
+
+ // Every node should be evenly used.
+ int averageLoadOnNode = maxPipelineCount *
+ HddsProtos.ReplicationFactor.THREE.getNumber() / healthyNodes.size();
+ for (DatanodeDetails node : healthyNodes) {
+ Assert.assertTrue(nodeManager.getPipelinesCount(node)
+ >= averageLoadOnNode);
+ }
+
+ // Should max out pipeline usage.
+ Assert.assertEquals(maxPipelineCount,
+ stateManager.getPipelines(HddsProtos.ReplicationType.RATIS).size());
+ }
+
+ @Test
public void testChooseNodeBasedOnRackAwareness() {
List<DatanodeDetails> healthyNodes = overWriteLocationInNodes(
nodeManager.getNodes(HddsProtos.NodeState.HEALTHY));
@@ -190,7 +259,8 @@
Assert.assertTrue(anchor.getNetworkLocation().equals(
randomNode.getNetworkLocation()));
- NetworkTopology topology = new NetworkTopologyImpl(new Configuration());
+ NetworkTopology topology =
+ new NetworkTopologyImpl(new OzoneConfiguration());
DatanodeDetails nextNode = placementPolicy.chooseNodeBasedOnRackAwareness(
nodesWithOutRackAwareness, new ArrayList<>(
PIPELINE_PLACEMENT_MAX_NODES_COUNT), topology, anchor);
@@ -229,7 +299,8 @@
};
private NetworkTopology createNetworkTopologyOnDifRacks() {
- NetworkTopology topology = new NetworkTopologyImpl(new Configuration());
+ NetworkTopology topology =
+ new NetworkTopologyImpl(new OzoneConfiguration());
for (Node n : NODES) {
topology.add(n);
}
@@ -297,6 +368,77 @@
Assert.assertTrue(thrown);
}
+ @Test
+ public void testValidatePlacementPolicyOK() {
+ cluster = initTopology();
+ nodeManager = new MockNodeManager(cluster, getNodesWithRackAwareness(),
+ false, PIPELINE_PLACEMENT_MAX_NODES_COUNT);
+ placementPolicy = new PipelinePlacementPolicy(
+ nodeManager, stateManager, conf);
+
+ List<DatanodeDetails> dns = new ArrayList<>();
+ dns.add(MockDatanodeDetails
+ .createDatanodeDetails("host1", "/rack1"));
+ dns.add(MockDatanodeDetails
+ .createDatanodeDetails("host2", "/rack1"));
+ dns.add(MockDatanodeDetails
+ .createDatanodeDetails("host3", "/rack2"));
+ for (DatanodeDetails dn : dns) {
+ cluster.add(dn);
+ }
+ ContainerPlacementStatus status =
+ placementPolicy.validateContainerPlacement(dns, 3);
+ assertTrue(status.isPolicySatisfied());
+ assertEquals(0, status.misReplicationCount());
+
+
+ List<DatanodeDetails> subSet = new ArrayList<>();
+ // Cut it down to two nodes, two racks
+ subSet.add(dns.get(0));
+ subSet.add(dns.get(2));
+ status = placementPolicy.validateContainerPlacement(subSet, 3);
+ assertTrue(status.isPolicySatisfied());
+ assertEquals(0, status.misReplicationCount());
+
+ // Cut it down to two nodes, one racks
+ subSet = new ArrayList<>();
+ subSet.add(dns.get(0));
+ subSet.add(dns.get(1));
+ status = placementPolicy.validateContainerPlacement(subSet, 3);
+ assertFalse(status.isPolicySatisfied());
+ assertEquals(1, status.misReplicationCount());
+
+ // One node, but only one replica
+ subSet = new ArrayList<>();
+ subSet.add(dns.get(0));
+ status = placementPolicy.validateContainerPlacement(subSet, 1);
+ assertTrue(status.isPolicySatisfied());
+ }
+
+ @Test
+ public void testValidatePlacementPolicySingleRackInCluster() {
+ cluster = initTopology();
+ nodeManager = new MockNodeManager(cluster, new ArrayList<>(),
+ false, PIPELINE_PLACEMENT_MAX_NODES_COUNT);
+ placementPolicy = new PipelinePlacementPolicy(
+ nodeManager, stateManager, conf);
+
+ List<DatanodeDetails> dns = new ArrayList<>();
+ dns.add(MockDatanodeDetails
+ .createDatanodeDetails("host1", "/rack1"));
+ dns.add(MockDatanodeDetails
+ .createDatanodeDetails("host2", "/rack1"));
+ dns.add(MockDatanodeDetails
+ .createDatanodeDetails("host3", "/rack1"));
+ for (DatanodeDetails dn : dns) {
+ cluster.add(dn);
+ }
+ ContainerPlacementStatus status =
+ placementPolicy.validateContainerPlacement(dns, 3);
+ assertTrue(status.isPolicySatisfied());
+ assertEquals(0, status.misReplicationCount());
+ }
+
private boolean checkDuplicateNodesUUID(List<DatanodeDetails> nodes) {
HashSet<UUID> uuids = nodes.stream().
map(DatanodeDetails::getUuid).
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
index ab23153..1b273fe 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
@@ -18,11 +18,6 @@
package org.apache.hadoop.hdds.scm.pipeline;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT;
-import static org.apache.hadoop.test.MetricsAsserts.getLongCounter;
-import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
-
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
@@ -30,36 +25,48 @@
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.TestUtils;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.MockNodeManager;
-import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
- .PipelineReportFromDatanode;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition;
+import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineReportFromDatanode;
import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.test.GenericTestUtils;
+
+import com.google.common.base.Supplier;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT;
+import static org.apache.hadoop.test.MetricsAsserts.getLongCounter;
+import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
import org.junit.After;
import org.junit.Assert;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
import org.junit.Before;
import org.junit.Test;
+
/**
* Test cases to verify PipelineManager.
*/
public class TestSCMPipelineManager {
private static MockNodeManager nodeManager;
private static File testDir;
- private static Configuration conf;
+ private static OzoneConfiguration conf;
+ private DBStore store;
@Before
public void setUp() throws Exception {
@@ -74,17 +81,25 @@
throw new IOException("Unable to create test directory path");
}
nodeManager = new MockNodeManager(true, 20);
+
+ store = DBStoreBuilder.createDBStore(conf, new SCMDBDefinition());
+
}
@After
- public void cleanup() {
+ public void cleanup() throws Exception {
+ store.close();
FileUtil.fullyDelete(testDir);
}
@Test
public void testPipelineReload() throws IOException {
SCMPipelineManager pipelineManager =
- new SCMPipelineManager(conf, nodeManager, new EventQueue());
+ new SCMPipelineManager(conf,
+ nodeManager,
+ SCMDBDefinition.PIPELINES.getTable(store),
+ new EventQueue());
+ pipelineManager.allowPipelineCreation();
PipelineProvider mockRatisProvider =
new MockRatisPipelineProvider(nodeManager,
pipelineManager.getStateManager(), conf);
@@ -103,7 +118,9 @@
// new pipeline manager should be able to load the pipelines from the db
pipelineManager =
- new SCMPipelineManager(conf, nodeManager, new EventQueue());
+ new SCMPipelineManager(conf, nodeManager,
+ SCMDBDefinition.PIPELINES.getTable(store), new EventQueue());
+ pipelineManager.allowPipelineCreation();
mockRatisProvider =
new MockRatisPipelineProvider(nodeManager,
pipelineManager.getStateManager(), conf);
@@ -133,7 +150,9 @@
@Test
public void testRemovePipeline() throws IOException {
SCMPipelineManager pipelineManager =
- new SCMPipelineManager(conf, nodeManager, new EventQueue());
+ new SCMPipelineManager(conf, nodeManager,
+ SCMDBDefinition.PIPELINES.getTable(store), new EventQueue());
+ pipelineManager.allowPipelineCreation();
PipelineProvider mockRatisProvider =
new MockRatisPipelineProvider(nodeManager,
pipelineManager.getStateManager(), conf);
@@ -151,10 +170,11 @@
// new pipeline manager should not be able to load removed pipelines
pipelineManager =
- new SCMPipelineManager(conf, nodeManager, new EventQueue());
+ new SCMPipelineManager(conf, nodeManager,
+ SCMDBDefinition.PIPELINES.getTable(store), new EventQueue());
try {
pipelineManager.getPipeline(pipeline.getId());
- Assert.fail("Pipeline should not have been retrieved");
+ fail("Pipeline should not have been retrieved");
} catch (IOException e) {
Assert.assertTrue(e.getMessage().contains("not found"));
}
@@ -167,7 +187,9 @@
public void testPipelineReport() throws IOException {
EventQueue eventQueue = new EventQueue();
SCMPipelineManager pipelineManager =
- new SCMPipelineManager(conf, nodeManager, eventQueue);
+ new SCMPipelineManager(conf, nodeManager,
+ SCMDBDefinition.PIPELINES.getTable(store), eventQueue);
+ pipelineManager.allowPipelineCreation();
PipelineProvider mockRatisProvider =
new MockRatisPipelineProvider(nodeManager,
pipelineManager.getStateManager(), conf);
@@ -218,7 +240,7 @@
try {
pipelineManager.getPipeline(pipeline.getId());
- Assert.fail("Pipeline should not have been retrieved");
+ fail("Pipeline should not have been retrieved");
} catch (IOException e) {
Assert.assertTrue(e.getMessage().contains("not found"));
}
@@ -232,7 +254,9 @@
MockNodeManager nodeManagerMock = new MockNodeManager(true,
20);
SCMPipelineManager pipelineManager =
- new SCMPipelineManager(conf, nodeManagerMock, new EventQueue());
+ new SCMPipelineManager(conf, nodeManagerMock,
+ SCMDBDefinition.PIPELINES.getTable(store), new EventQueue());
+ pipelineManager.allowPipelineCreation();
PipelineProvider mockRatisProvider =
new MockRatisPipelineProvider(nodeManagerMock,
pipelineManager.getStateManager(), conf);
@@ -243,7 +267,7 @@
SCMPipelineMetrics.class.getSimpleName());
long numPipelineAllocated = getLongCounter("NumPipelineAllocated",
metrics);
- Assert.assertTrue(numPipelineAllocated == 0);
+ Assert.assertEquals(0, numPipelineAllocated);
// 3 DNs are unhealthy.
// Create 5 pipelines (Use up 15 Datanodes)
@@ -257,17 +281,17 @@
metrics = getMetrics(
SCMPipelineMetrics.class.getSimpleName());
numPipelineAllocated = getLongCounter("NumPipelineAllocated", metrics);
- Assert.assertTrue(numPipelineAllocated == 5);
+ Assert.assertEquals(5, numPipelineAllocated);
long numPipelineCreateFailed = getLongCounter(
"NumPipelineCreationFailed", metrics);
- Assert.assertTrue(numPipelineCreateFailed == 0);
+ Assert.assertEquals(0, numPipelineCreateFailed);
//This should fail...
try {
pipelineManager.createPipeline(HddsProtos.ReplicationType.RATIS,
HddsProtos.ReplicationFactor.THREE);
- Assert.fail();
+ fail();
} catch (SCMException ioe) {
// pipeline creation failed this time.
Assert.assertEquals(SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE,
@@ -277,11 +301,11 @@
metrics = getMetrics(
SCMPipelineMetrics.class.getSimpleName());
numPipelineAllocated = getLongCounter("NumPipelineAllocated", metrics);
- Assert.assertTrue(numPipelineAllocated == 5);
+ Assert.assertEquals(5, numPipelineAllocated);
numPipelineCreateFailed = getLongCounter(
"NumPipelineCreationFailed", metrics);
- Assert.assertTrue(numPipelineCreateFailed == 1);
+ Assert.assertEquals(1, numPipelineCreateFailed);
// clean up
pipelineManager.close();
@@ -290,7 +314,9 @@
@Test
public void testActivateDeactivatePipeline() throws IOException {
final SCMPipelineManager pipelineManager =
- new SCMPipelineManager(conf, nodeManager, new EventQueue());
+ new SCMPipelineManager(conf, nodeManager,
+ SCMDBDefinition.PIPELINES.getTable(store), new EventQueue());
+ pipelineManager.allowPipelineCreation();
final PipelineProvider mockRatisProvider =
new MockRatisPipelineProvider(nodeManager,
pipelineManager.getStateManager(), conf);
@@ -337,12 +363,16 @@
public void testPipelineOpenOnlyWhenLeaderReported() throws Exception {
EventQueue eventQueue = new EventQueue();
SCMPipelineManager pipelineManager =
- new SCMPipelineManager(conf, nodeManager, eventQueue);
+ new SCMPipelineManager(conf, nodeManager,
+ SCMDBDefinition.PIPELINES.getTable(store), eventQueue);
+ pipelineManager.allowPipelineCreation();
PipelineProvider mockRatisProvider =
new MockRatisPipelineProvider(nodeManager,
pipelineManager.getStateManager(), conf);
pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS,
mockRatisProvider);
+ pipelineManager.onMessage(
+ new SCMSafeModeManager.SafeModeStatus(true, true), null);
Pipeline pipeline = pipelineManager
.createPipeline(HddsProtos.ReplicationType.RATIS,
HddsProtos.ReplicationFactor.THREE);
@@ -350,7 +380,8 @@
pipelineManager.close();
// new pipeline manager loads the pipelines from the db in ALLOCATED state
pipelineManager =
- new SCMPipelineManager(conf, nodeManager, eventQueue);
+ new SCMPipelineManager(conf, nodeManager,
+ SCMDBDefinition.PIPELINES.getTable(store), eventQueue);
mockRatisProvider =
new MockRatisPipelineProvider(nodeManager,
pipelineManager.getStateManager(), conf);
@@ -395,7 +426,9 @@
EventQueue eventQueue = new EventQueue();
final SCMPipelineManager pipelineManager =
- new SCMPipelineManager(conf, nodeManager, eventQueue);
+ new SCMPipelineManager(conf, nodeManager,
+ SCMDBDefinition.PIPELINES.getTable(store), eventQueue);
+ pipelineManager.allowPipelineCreation();
final PipelineProvider ratisProvider = new MockRatisPipelineProvider(
nodeManager, pipelineManager.getStateManager(), conf, eventQueue,
false);
@@ -407,8 +440,8 @@
.createPipeline(HddsProtos.ReplicationType.RATIS,
HddsProtos.ReplicationFactor.THREE);
// At this point, pipeline is not at OPEN stage.
- Assert.assertEquals(pipeline.getPipelineState(),
- Pipeline.PipelineState.ALLOCATED);
+ Assert.assertEquals(Pipeline.PipelineState.ALLOCATED,
+ pipeline.getPipelineState());
// pipeline should be seen in pipelineManager as ALLOCATED.
Assert.assertTrue(pipelineManager
@@ -427,6 +460,87 @@
pipelineManager.close();
}
+ @Test
+ public void testPipelineNotCreatedUntilSafeModePrecheck()
+ throws IOException, TimeoutException, InterruptedException {
+ // No timeout for pipeline scrubber.
+ conf.setTimeDuration(
+ OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT, -1,
+ TimeUnit.MILLISECONDS);
+
+ EventQueue eventQueue = new EventQueue();
+ SCMPipelineManager pipelineManager =
+ new SCMPipelineManager(conf, nodeManager,
+ SCMDBDefinition.PIPELINES.getTable(store), eventQueue);
+ final PipelineProvider ratisProvider = new MockRatisPipelineProvider(
+ nodeManager, pipelineManager.getStateManager(), conf, eventQueue,
+ false);
+
+ pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS,
+ ratisProvider);
+
+ try {
+ Pipeline pipeline = pipelineManager
+ .createPipeline(HddsProtos.ReplicationType.RATIS,
+ HddsProtos.ReplicationFactor.THREE);
+ fail("Pipelines should not have been created");
+ } catch (IOException e) {
+ // expected
+ }
+
+ // Ensure a pipeline of factor ONE can be created - no exceptions should be
+ // raised.
+ Pipeline pipeline = pipelineManager
+ .createPipeline(HddsProtos.ReplicationType.RATIS,
+ HddsProtos.ReplicationFactor.ONE);
+
+ // Simulate safemode check exiting.
+ pipelineManager.onMessage(
+ new SCMSafeModeManager.SafeModeStatus(true, true), null);
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return pipelineManager.getPipelines().size() != 0;
+ }
+ }, 100, 10000);
+ pipelineManager.close();
+ }
+
+ @Test
+ public void testSafeModeUpdatedOnSafemodeExit()
+ throws IOException, TimeoutException, InterruptedException {
+ // No timeout for pipeline scrubber.
+ conf.setTimeDuration(
+ OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT, -1,
+ TimeUnit.MILLISECONDS);
+
+ EventQueue eventQueue = new EventQueue();
+ SCMPipelineManager pipelineManager =
+ new SCMPipelineManager(conf, nodeManager,
+ SCMDBDefinition.PIPELINES.getTable(store), eventQueue);
+ final PipelineProvider ratisProvider = new MockRatisPipelineProvider(
+ nodeManager, pipelineManager.getStateManager(), conf, eventQueue,
+ false);
+
+ pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS,
+ ratisProvider);
+
+ assertEquals(true, pipelineManager.getSafeModeStatus());
+ assertEquals(false, pipelineManager.isPipelineCreationAllowed());
+ // First pass pre-check as true, but safemode still on
+ pipelineManager.onMessage(
+ new SCMSafeModeManager.SafeModeStatus(true, true), null);
+ assertEquals(true, pipelineManager.getSafeModeStatus());
+ assertEquals(true, pipelineManager.isPipelineCreationAllowed());
+
+ // Then also turn safemode off
+ pipelineManager.onMessage(
+ new SCMSafeModeManager.SafeModeStatus(false, true), null);
+ assertEquals(false, pipelineManager.getSafeModeStatus());
+ assertEquals(true, pipelineManager.isPipelineCreationAllowed());
+ pipelineManager.close();
+ }
+
private void sendPipelineReport(DatanodeDetails dn,
Pipeline pipeline, PipelineReportHandler pipelineReportHandler,
boolean isLeader, EventQueue eventQueue) {
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSimplePipelineProvider.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSimplePipelineProvider.java
index d9a42f2..fe3fb79 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSimplePipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSimplePipelineProvider.java
@@ -44,7 +44,7 @@
public void init() throws Exception {
nodeManager = new MockNodeManager(true, 10);
stateManager = new PipelineStateManager();
- provider = new SimplePipelineProvider(nodeManager);
+ provider = new SimplePipelineProvider(nodeManager, stateManager);
}
@Test
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java
index 890973f..700479d 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java
@@ -18,6 +18,11 @@
package org.apache.hadoop.hdds.scm.safemode;
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -26,21 +31,20 @@
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.MockNodeManager;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition;
+import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider;
-import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider;
import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
import org.apache.hadoop.test.GenericTestUtils;
+
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.UUID;
-
/**
* This class tests HealthyPipelineSafeMode rule.
*/
@@ -49,7 +53,7 @@
@Test
public void testHealthyPipelineSafeModeRuleWithNoPipelines()
throws Exception {
-
+ DBStore store = null;
String storageDir = GenericTestUtils.getTempPath(
TestHealthyPipelineSafeModeRule.class.getName() + UUID.randomUUID());
try {
@@ -65,9 +69,9 @@
HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true);
config.setBoolean(
HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false);
-
+ store = DBStoreBuilder.createDBStore(config, new SCMDBDefinition());
SCMPipelineManager pipelineManager = new SCMPipelineManager(config,
- nodeManager, eventQueue);
+ nodeManager, SCMDBDefinition.PIPELINES.getTable(store), eventQueue);
PipelineProvider mockRatisProvider =
new MockRatisPipelineProvider(nodeManager,
pipelineManager.getStateManager(), config);
@@ -82,16 +86,16 @@
// This should be immediately satisfied, as no pipelines are there yet.
Assert.assertTrue(healthyPipelineSafeModeRule.validate());
} finally {
+ store.close();
FileUtil.fullyDelete(new File(storageDir));
}
}
@Test
public void testHealthyPipelineSafeModeRuleWithPipelines() throws Exception {
-
String storageDir = GenericTestUtils.getTempPath(
TestHealthyPipelineSafeModeRule.class.getName() + UUID.randomUUID());
-
+ DBStore store = null;
try {
EventQueue eventQueue = new EventQueue();
List<ContainerInfo> containers =
@@ -109,8 +113,10 @@
config.setBoolean(
HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false);
+ store = DBStoreBuilder.createDBStore(config, new SCMDBDefinition());
SCMPipelineManager pipelineManager = new SCMPipelineManager(config,
- nodeManager, eventQueue);
+ nodeManager, SCMDBDefinition.PIPELINES.getTable(store), eventQueue);
+ pipelineManager.allowPipelineCreation();
PipelineProvider mockRatisProvider =
new MockRatisPipelineProvider(nodeManager,
@@ -152,6 +158,7 @@
GenericTestUtils.waitFor(() -> healthyPipelineSafeModeRule.validate(),
1000, 5000);
} finally {
+ store.close();
FileUtil.fullyDelete(new File(storageDir));
}
}
@@ -163,6 +170,7 @@
String storageDir = GenericTestUtils.getTempPath(
TestHealthyPipelineSafeModeRule.class.getName() + UUID.randomUUID());
+ DBStore store = null;
try {
EventQueue eventQueue = new EventQueue();
@@ -182,8 +190,11 @@
config.setBoolean(
HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false);
+ store = DBStoreBuilder.createDBStore(config, new SCMDBDefinition());
SCMPipelineManager pipelineManager = new SCMPipelineManager(config,
- nodeManager, eventQueue);
+ nodeManager, SCMDBDefinition.PIPELINES.getTable(store), eventQueue);
+
+ pipelineManager.allowPipelineCreation();
PipelineProvider mockRatisProvider =
new MockRatisPipelineProvider(nodeManager,
pipelineManager.getStateManager(), config, true);
@@ -233,6 +244,7 @@
1000, 5000);
} finally {
+ store.close();
FileUtil.fullyDelete(new File(storageDir));
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java
index 0fa5eae..c1f09fa 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java
@@ -17,28 +17,32 @@
package org.apache.hadoop.hdds.scm.safemode;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.HddsTestUtils;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.MockNodeManager;
-import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition;
+import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider;
import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
import org.apache.hadoop.test.GenericTestUtils;
+
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.List;
-
/**
* This class tests OneReplicaPipelineSafeModeRule.
*/
@@ -50,7 +54,6 @@
private SCMPipelineManager pipelineManager;
private EventQueue eventQueue;
-
private void setup(int nodes, int pipelineFactorThreeCount,
int pipelineFactorOneCount) throws Exception {
OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
@@ -66,9 +69,15 @@
MockNodeManager mockNodeManager = new MockNodeManager(true, nodes);
eventQueue = new EventQueue();
+
+ DBStore dbStore =
+ DBStoreBuilder.createDBStore(ozoneConfiguration, new SCMDBDefinition());
+
pipelineManager =
new SCMPipelineManager(ozoneConfiguration, mockNodeManager,
+ SCMDBDefinition.PIPELINES.getTable(dbStore),
eventQueue);
+ pipelineManager.allowPipelineCreation();
PipelineProvider mockRatisProvider =
new MockRatisPipelineProvider(mockNodeManager,
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java
index d9cff01..b86d354 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java
@@ -17,17 +17,16 @@
*/
package org.apache.hadoop.hdds.scm.safemode;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
import java.io.File;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -35,16 +34,27 @@
import org.apache.hadoop.hdds.scm.HddsTestUtils;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.MockNodeManager;
-import org.apache.hadoop.hdds.scm.pipeline.*;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition;
+import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider;
import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
+import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager.SafeModeStatus;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
import org.apache.hadoop.test.GenericTestUtils;
+
+import org.junit.After;
import org.junit.Assert;
-import org.junit.BeforeClass;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -57,7 +67,7 @@
private static EventQueue queue;
private SCMSafeModeManager scmSafeModeManager;
- private static Configuration config;
+ private static OzoneConfiguration config;
private List<ContainerInfo> containers = Collections.emptyList();
@Rule
@@ -66,14 +76,30 @@
@Rule
public final TemporaryFolder tempDir = new TemporaryFolder();
- @BeforeClass
- public static void setUp() {
+ private DBStore dbStore;
+
+ @Before
+ public void setUp() {
queue = new EventQueue();
config = new OzoneConfiguration();
config.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION,
false);
}
+ @Before
+ public void initDbStore() throws IOException {
+ config.set(HddsConfigKeys.OZONE_METADATA_DIRS,
+ tempDir.newFolder().getAbsolutePath());
+ dbStore = DBStoreBuilder.createDBStore(config, new SCMDBDefinition());
+ }
+
+ @After
+ public void destroyDbStore() throws Exception {
+ if (dbStore != null) {
+ dbStore.close();
+ }
+ }
+
@Test
public void testSafeModeState() throws Exception {
// Test 1: test for 0 containers
@@ -121,6 +147,59 @@
}
@Test
+ public void testDelayedEventNotification() throws Exception {
+
+ List<SafeModeStatus> delayedSafeModeEvents = new ArrayList<>();
+ List<SafeModeStatus> safeModeEvents = new ArrayList<>();
+
+ //given
+ EventQueue eventQueue = new EventQueue();
+ eventQueue.addHandler(SCMEvents.SAFE_MODE_STATUS,
+ (safeModeStatus, publisher) -> safeModeEvents.add(safeModeStatus));
+ eventQueue.addHandler(SCMEvents.DELAYED_SAFE_MODE_STATUS,
+ (safeModeStatus, publisher) -> delayedSafeModeEvents
+ .add(safeModeStatus));
+
+ OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
+ ozoneConfiguration
+ .setTimeDuration(HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT,
+ 3, TimeUnit.SECONDS);
+ ozoneConfiguration
+ .setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false);
+
+ scmSafeModeManager = new SCMSafeModeManager(
+ ozoneConfiguration, containers, null, eventQueue);
+
+ //when
+ scmSafeModeManager.setInSafeMode(true);
+ scmSafeModeManager.setPreCheckComplete(true);
+
+ scmSafeModeManager.emitSafeModeStatus();
+ eventQueue.processAll(1000L);
+
+ //then
+ Assert.assertEquals(1, delayedSafeModeEvents.size());
+ Assert.assertEquals(1, safeModeEvents.size());
+
+ //when
+ scmSafeModeManager.setInSafeMode(false);
+ scmSafeModeManager.setPreCheckComplete(true);
+
+ scmSafeModeManager.emitSafeModeStatus();
+ eventQueue.processAll(1000L);
+
+ //then
+ Assert.assertEquals(2, safeModeEvents.size());
+ //delayed messages are not yet sent (unless JVM is paused for 3 seconds)
+ Assert.assertEquals(1, delayedSafeModeEvents.size());
+
+ //event will be triggered after 3 seconds (see previous config)
+ GenericTestUtils.waitFor(() -> delayedSafeModeEvents.size() == 2,
+ 300,
+ 6000);
+
+ }
+ @Test
public void testSafeModeExitRule() throws Exception {
containers = new ArrayList<>();
int numContainers = 100;
@@ -181,12 +260,32 @@
}
@Test
- public void testSafeModeExitRuleWithPipelineAvailabilityCheck()
- throws Exception{
+ public void testSafeModeExitRuleWithPipelineAvailabilityCheck1()
+ throws Exception {
testSafeModeExitRuleWithPipelineAvailabilityCheck(100, 30, 8, 0.90, 1);
+ }
+
+ @Test
+ public void testSafeModeExitRuleWithPipelineAvailabilityCheck2()
+ throws Exception {
testSafeModeExitRuleWithPipelineAvailabilityCheck(100, 90, 22, 0.10, 0.9);
+ }
+
+ @Test
+ public void testSafeModeExitRuleWithPipelineAvailabilityCheck3()
+ throws Exception {
testSafeModeExitRuleWithPipelineAvailabilityCheck(100, 30, 8, 0, 0.9);
+ }
+
+ @Test
+ public void testSafeModeExitRuleWithPipelineAvailabilityCheck4()
+ throws Exception {
testSafeModeExitRuleWithPipelineAvailabilityCheck(100, 90, 22, 0, 0);
+ }
+
+ @Test
+ public void testSafeModeExitRuleWithPipelineAvailabilityCheck5()
+ throws Exception {
testSafeModeExitRuleWithPipelineAvailabilityCheck(100, 90, 22, 0, 0.5);
}
@@ -198,7 +297,7 @@
0.9);
MockNodeManager mockNodeManager = new MockNodeManager(true, 10);
PipelineManager pipelineManager = new SCMPipelineManager(conf,
- mockNodeManager, queue);
+ mockNodeManager, SCMDBDefinition.PIPELINES.getTable(dbStore), queue);
scmSafeModeManager = new SCMSafeModeManager(
conf, containers, pipelineManager, queue);
fail("testFailWithIncorrectValueForHealthyPipelinePercent");
@@ -216,7 +315,7 @@
200);
MockNodeManager mockNodeManager = new MockNodeManager(true, 10);
PipelineManager pipelineManager = new SCMPipelineManager(conf,
- mockNodeManager, queue);
+ mockNodeManager, SCMDBDefinition.PIPELINES.getTable(dbStore), queue);
scmSafeModeManager = new SCMSafeModeManager(
conf, containers, pipelineManager, queue);
fail("testFailWithIncorrectValueForOneReplicaPipelinePercent");
@@ -233,7 +332,7 @@
conf.setDouble(HddsConfigKeys.HDDS_SCM_SAFEMODE_THRESHOLD_PCT, -1.0);
MockNodeManager mockNodeManager = new MockNodeManager(true, 10);
PipelineManager pipelineManager = new SCMPipelineManager(conf,
- mockNodeManager, queue);
+ mockNodeManager, SCMDBDefinition.PIPELINES.getTable(dbStore), queue);
scmSafeModeManager = new SCMSafeModeManager(
conf, containers, pipelineManager, queue);
fail("testFailWithIncorrectValueForSafeModePercent");
@@ -257,13 +356,13 @@
MockNodeManager mockNodeManager = new MockNodeManager(true, nodeCount);
SCMPipelineManager pipelineManager = new SCMPipelineManager(conf,
- mockNodeManager, queue);
+ mockNodeManager, SCMDBDefinition.PIPELINES.getTable(dbStore), queue);
PipelineProvider mockRatisProvider =
new MockRatisPipelineProvider(mockNodeManager,
pipelineManager.getStateManager(), config, true);
pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS,
mockRatisProvider);
-
+ pipelineManager.allowPipelineCreation();
for (int i=0; i < pipelineCount; i++) {
pipelineManager.createPipeline(HddsProtos.ReplicationType.RATIS,
@@ -474,13 +573,14 @@
HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true);
SCMPipelineManager pipelineManager = new SCMPipelineManager(config,
- nodeManager, queue);
+ nodeManager, SCMDBDefinition.PIPELINES.getTable(dbStore), queue);
PipelineProvider mockRatisProvider =
new MockRatisPipelineProvider(nodeManager,
pipelineManager.getStateManager(), config, true);
pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS,
mockRatisProvider);
+ pipelineManager.allowPipelineCreation();
Pipeline pipeline = pipelineManager.createPipeline(
HddsProtos.ReplicationType.RATIS,
@@ -506,4 +606,102 @@
FileUtil.fullyDelete(new File(storageDir));
}
}
+
+ @Test
+ public void testPipelinesNotCreatedUntilPreCheckPasses()
+ throws Exception {
+ int numOfDns = 5;
+ // enable pipeline check
+ config.setBoolean(
+ HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true);
+ config.setInt(HddsConfigKeys.HDDS_SCM_SAFEMODE_MIN_DATANODE, numOfDns);
+ config.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION,
+ true);
+
+ MockNodeManager nodeManager = new MockNodeManager(true, numOfDns);
+ String storageDir = GenericTestUtils.getTempPath(
+ TestSCMSafeModeManager.class.getName() + UUID.randomUUID());
+ config.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir);
+ // enable pipeline check
+ config.setBoolean(
+ HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true);
+
+ SCMPipelineManager pipelineManager = new SCMPipelineManager(config,
+ nodeManager, SCMDBDefinition.PIPELINES.getTable(dbStore), queue);
+
+
+ PipelineProvider mockRatisProvider =
+ new MockRatisPipelineProvider(nodeManager,
+ pipelineManager.getStateManager(), config, true);
+ pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS,
+ mockRatisProvider);
+
+ SafeModeEventHandler smHandler = new SafeModeEventHandler();
+ queue.addHandler(SCMEvents.SAFE_MODE_STATUS, smHandler);
+ scmSafeModeManager = new SCMSafeModeManager(
+ config, containers, pipelineManager, queue);
+
+ // Assert SCM is in Safe mode.
+ assertTrue(scmSafeModeManager.getInSafeMode());
+
+ // Register all DataNodes except last one and assert SCM is in safe mode.
+ for (int i = 0; i < numOfDns - 1; i++) {
+ queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+ HddsTestUtils.createNodeRegistrationContainerReport(containers));
+ assertTrue(scmSafeModeManager.getInSafeMode());
+ assertFalse(scmSafeModeManager.getPreCheckComplete());
+ }
+ queue.processAll(5000);
+ Assert.assertEquals(0, smHandler.getInvokedCount());
+
+ // Register last DataNode and check that the SafeModeEvent gets fired, but
+ // safemode is still enabled with preCheck completed.
+ queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+ HddsTestUtils.createNodeRegistrationContainerReport(containers));
+ queue.processAll(5000);
+
+ Assert.assertEquals(1, smHandler.getInvokedCount());
+ Assert.assertEquals(true, smHandler.getPreCheckComplete());
+ Assert.assertEquals(true, smHandler.getIsInSafeMode());
+
+ // Create a pipeline and ensure safemode is exited.
+ pipelineManager.allowPipelineCreation();
+ Pipeline pipeline = pipelineManager.createPipeline(
+ HddsProtos.ReplicationType.RATIS,
+ HddsProtos.ReplicationFactor.THREE);
+ firePipelineEvent(pipelineManager, pipeline);
+
+ queue.processAll(5000);
+ Assert.assertEquals(2, smHandler.getInvokedCount());
+ Assert.assertEquals(true, smHandler.getPreCheckComplete());
+ Assert.assertEquals(false, smHandler.getIsInSafeMode());
+ }
+
+ private static class SafeModeEventHandler
+ implements EventHandler<SCMSafeModeManager.SafeModeStatus> {
+
+ private AtomicInteger invokedCount = new AtomicInteger(0);
+ private AtomicBoolean preCheckComplete = new AtomicBoolean(false);
+ private AtomicBoolean isInSafeMode = new AtomicBoolean(true);
+
+ public int getInvokedCount() {
+ return invokedCount.get();
+ }
+
+ public boolean getPreCheckComplete() {
+ return preCheckComplete.get();
+ }
+
+ public boolean getIsInSafeMode() {
+ return isInSafeMode.get();
+ }
+
+ @Override
+ public void onMessage(SCMSafeModeManager.SafeModeStatus safeModeStatus,
+ EventPublisher publisher) {
+ invokedCount.incrementAndGet();
+ preCheckComplete.set(safeModeStatus.isPreCheckComplete());
+ isInSafeMode.set(safeModeStatus.isInSafeMode());
+ }
+ }
}
\ No newline at end of file
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeHandler.java
deleted file mode 100644
index ccd06e9..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeHandler.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.safemode;
-
-import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.scm.block.BlockManager;
-import org.apache.hadoop.hdds.scm.block.BlockManagerImpl;
-import org.apache.hadoop.hdds.scm.container.ContainerManager;
-import org.apache.hadoop.hdds.scm.container.ReplicationManager;
-import org.apache.hadoop.hdds.scm.container.ReplicationManager.ReplicationManagerConfiguration;
-import org.apache.hadoop.hdds.scm.PlacementPolicy;
-import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
-import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
-import org.apache.hadoop.hdds.scm.server.SCMClientProtocolServer;
-import org.apache.hadoop.hdds.server.events.EventQueue;
-import org.apache.hadoop.ozone.lock.LockManager;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.Assert;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-import java.util.HashSet;
-
-/**
- * Tests SafeModeHandler behavior.
- */
-public class TestSafeModeHandler {
-
-
- private OzoneConfiguration configuration;
- private SCMClientProtocolServer scmClientProtocolServer;
- private ReplicationManager replicationManager;
- private BlockManager blockManager;
- private SafeModeHandler safeModeHandler;
- private EventQueue eventQueue;
- private SCMSafeModeManager.SafeModeStatus safeModeStatus;
- private PipelineManager scmPipelineManager;
-
- public void setup(boolean enabled) {
- configuration = new OzoneConfiguration();
- configuration.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_ENABLED,
- enabled);
- configuration.set(HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT,
- "3s");
- scmClientProtocolServer =
- Mockito.mock(SCMClientProtocolServer.class);
- eventQueue = new EventQueue();
- final ContainerManager containerManager =
- Mockito.mock(ContainerManager.class);
- Mockito.when(containerManager.getContainerIDs())
- .thenReturn(new HashSet<>());
- replicationManager = new ReplicationManager(
- new ReplicationManagerConfiguration(),
- containerManager, Mockito.mock(PlacementPolicy.class),
- eventQueue, new LockManager(configuration));
- scmPipelineManager = Mockito.mock(SCMPipelineManager.class);
- blockManager = Mockito.mock(BlockManagerImpl.class);
- safeModeHandler = new SafeModeHandler(configuration);
- safeModeHandler.notifyImmediately(scmClientProtocolServer, blockManager);
- safeModeHandler.notifyAfterDelay(replicationManager, scmPipelineManager);
-
- eventQueue.addHandler(SCMEvents.SAFE_MODE_STATUS, safeModeHandler);
- safeModeStatus = new SCMSafeModeManager.SafeModeStatus(false);
-
- }
-
- @Test
- public void testSafeModeHandlerWithSafeModeEnabled() throws Exception {
- setup(true);
-
- Assert.assertTrue(safeModeHandler.getSafeModeStatus());
-
- eventQueue.fireEvent(SCMEvents.SAFE_MODE_STATUS, safeModeStatus);
-
- GenericTestUtils.waitFor(() -> !safeModeHandler.getSafeModeStatus(),
- 1000, 5000);
-
- Assert.assertFalse(scmClientProtocolServer.getSafeModeStatus());
- Assert.assertFalse(((BlockManagerImpl) blockManager).isScmInSafeMode());
- GenericTestUtils.waitFor(() ->
- replicationManager.isRunning(), 1000, 5000);
- }
-
-
- @Test
- public void testSafeModeHandlerWithSafeModeDisbaled() throws Exception{
-
- setup(false);
-
- Assert.assertFalse(safeModeHandler.getSafeModeStatus());
-
- eventQueue.fireEvent(SCMEvents.SAFE_MODE_STATUS, safeModeStatus);
-
- Assert.assertFalse(safeModeHandler.getSafeModeStatus());
- Assert.assertFalse(scmClientProtocolServer.getSafeModeStatus());
- Assert.assertFalse(((BlockManagerImpl) blockManager).isScmInSafeMode());
- GenericTestUtils.waitFor(() ->
- replicationManager.isRunning(), 1000, 5000);
- }
-}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMClientProtocolServer.java
deleted file mode 100644
index 8bdeba4..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMClientProtocolServer.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.server;
-
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_DEFAULT;
-
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
-import org.apache.hadoop.hdds.scm.block.BlockManager;
-import org.apache.hadoop.hdds.scm.block.BlockManagerImpl;
-import org.apache.hadoop.hdds.scm.safemode.SafeModeHandler;
-import org.apache.hadoop.hdds.scm.container.ReplicationManager;
-import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
-import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
-import org.apache.hadoop.hdds.server.events.EventQueue;
-import org.apache.hadoop.test.LambdaTestUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-/**
- * Test class for @{@link SCMClientProtocolServer}.
- * */
-public class TestSCMClientProtocolServer {
- private SCMClientProtocolServer scmClientProtocolServer;
- private OzoneConfiguration config;
- private EventQueue eventQueue;
-
- @Before
- public void setUp() throws Exception {
- config = new OzoneConfiguration();
- config.set(OZONE_SCM_CLIENT_ADDRESS_KEY,
- OZONE_SCM_CLIENT_BIND_HOST_DEFAULT + ":0");
- eventQueue = new EventQueue();
- scmClientProtocolServer = new SCMClientProtocolServer(config, null);
- BlockManager blockManager = Mockito.mock(BlockManagerImpl.class);
- ReplicationManager replicationManager =
- Mockito.mock(ReplicationManager.class);
- PipelineManager pipelineManager = Mockito.mock(SCMPipelineManager.class);
- SafeModeHandler safeModeHandler = new SafeModeHandler(config);
- safeModeHandler.notifyImmediately(scmClientProtocolServer, blockManager);
- safeModeHandler.notifyAfterDelay(replicationManager, pipelineManager);
- eventQueue.addHandler(SCMEvents.SAFE_MODE_STATUS, safeModeHandler);
- }
-
- @After
- public void tearDown() throws Exception {
- }
-
- @Test
- public void testAllocateContainerFailureInSafeMode() throws Exception {
- LambdaTestUtils.intercept(SCMException.class,
- "SafeModePrecheck failed for allocateContainer", () -> {
- scmClientProtocolServer.allocateContainer(
- ReplicationType.STAND_ALONE, ReplicationFactor.ONE, "");
- });
- }
-}
\ No newline at end of file
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
index bd89b88..663ac8c 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
@@ -22,7 +22,6 @@
import java.util.Map;
import java.util.UUID;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -83,7 +82,7 @@
private static RPC.Server scmServer;
private static ScmTestMock scmServerImpl;
private static File testDir;
- private static Configuration config;
+ private static OzoneConfiguration config;
@AfterClass
public static void tearDown() throws Exception {
@@ -292,7 +291,7 @@
private EndpointStateMachine registerTaskHelper(InetSocketAddress scmAddress,
int rpcTimeout, boolean clearDatanodeDetails) throws Exception {
- Configuration conf = SCMTestUtils.getConf();
+ OzoneConfiguration conf = SCMTestUtils.getConf();
EndpointStateMachine rpcEndPoint =
createEndpoint(conf,
scmAddress, rpcTimeout);
@@ -453,7 +452,7 @@
private StateContext heartbeatTaskHelper(InetSocketAddress scmAddress,
int rpcTimeout) throws Exception {
- Configuration conf = SCMTestUtils.getConf();
+ OzoneConfiguration conf = SCMTestUtils.getConf();
conf.set(DFS_DATANODE_DATA_DIR_KEY, testDir.getAbsolutePath());
conf.set(OZONE_METADATA_DIRS, testDir.getAbsolutePath());
// Mini Ozone cluster will not come up if the port is not true, since
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java
index f0b1cbb..a0cf957 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java
@@ -16,27 +16,24 @@
*/
package org.apache.hadoop.ozone.container.placement;
-import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.scm.container.MockNodeManager;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms
- .SCMContainerPlacementCapacity;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms
- .SCMContainerPlacementRandom;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.junit.Assert;
-import org.junit.Test;
-
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
-import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState
- .HEALTHY;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.container.MockNodeManager;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRandom;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.OzoneConsts;
+
+import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY;
+import org.junit.Assert;
import static org.junit.Assert.assertEquals;
+import org.junit.Test;
/**
* Asserts that allocation strategy works as expected.
@@ -80,10 +77,11 @@
.getStandardDeviation(), 0.001);
SCMContainerPlacementCapacity capacityPlacer = new
- SCMContainerPlacementCapacity(nodeManagerCapacity, new Configuration(),
+ SCMContainerPlacementCapacity(nodeManagerCapacity,
+ new OzoneConfiguration(),
null, true, null);
SCMContainerPlacementRandom randomPlacer = new
- SCMContainerPlacementRandom(nodeManagerRandom, new Configuration(),
+ SCMContainerPlacementRandom(nodeManagerRandom, new OzoneConfiguration(),
null, true, null);
for (int x = 0; x < opsCount; x++) {
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java
new file mode 100644
index 0000000..cebedf2
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java
@@ -0,0 +1,204 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.scm.node;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.UUID;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto;
+import org.apache.hadoop.hdds.scm.TestUtils;
+import org.apache.hadoop.hdds.scm.net.NetworkTopology;
+import org.apache.hadoop.hdds.scm.node.SCMNodeManager;
+import org.apache.hadoop.hdds.scm.node.SCMNodeMetrics;
+import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
+import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+
+import static org.apache.hadoop.test.MetricsAsserts.assertGauge;
+import static org.apache.hadoop.test.MetricsAsserts.getLongCounter;
+import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import static org.junit.Assert.assertEquals;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+/**
+ * Test cases to verify the metrics exposed by SCMNodeManager.
+ */
+public class TestSCMNodeMetrics {
+
+ private static SCMNodeManager nodeManager;
+
+ private static DatanodeDetails registeredDatanode;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+
+ OzoneConfiguration source = new OzoneConfiguration();
+ EventQueue publisher = new EventQueue();
+ SCMStorageConfig config =
+ new SCMStorageConfig(NodeType.DATANODE, new File("/tmp"), "storage");
+ nodeManager = new SCMNodeManager(source, config, publisher,
+ Mockito.mock(NetworkTopology.class));
+
+ registeredDatanode = DatanodeDetails.newBuilder()
+ .setHostName("localhost")
+ .setIpAddress("127.0.0.1")
+ .setUuid(UUID.randomUUID().toString())
+ .build();
+
+ nodeManager.register(registeredDatanode, createNodeReport(),
+ PipelineReportsProto.newBuilder().build());
+
+ }
+
+ @AfterClass
+ public static void teardown() throws IOException {
+ nodeManager.close();
+ }
+
+ /**
+ * Verifies heartbeat processing count.
+ *
+ * @throws InterruptedException
+ */
+ @Test
+ public void testHBProcessing() throws InterruptedException {
+ long hbProcessed = getCounter("NumHBProcessed");
+
+ NodeReportProto nodeReport = createNodeReport();
+
+ nodeManager.processHeartbeat(registeredDatanode);
+
+ assertEquals("NumHBProcessed", hbProcessed + 1,
+ getCounter("NumHBProcessed"));
+ }
+
+ /**
+ * Verifies heartbeat processing failure count.
+ */
+ @Test
+ public void testHBProcessingFailure() {
+
+ long hbProcessedFailed = getCounter("NumHBProcessingFailed");
+
+ nodeManager.processHeartbeat(MockDatanodeDetails
+ .randomDatanodeDetails());
+
+ assertEquals("NumHBProcessingFailed", hbProcessedFailed + 1,
+ getCounter("NumHBProcessingFailed"));
+ }
+
+ /**
+ * Verifies node report processing count.
+ *
+ * @throws InterruptedException
+ */
+ @Test
+ public void testNodeReportProcessing() throws InterruptedException {
+
+ long nrProcessed = getCounter("NumNodeReportProcessed");
+
+ StorageReportProto storageReport =
+ TestUtils.createStorageReport(registeredDatanode.getUuid(), "/tmp", 100,
+ 10, 90,
+ null);
+ NodeReportProto nodeReport = NodeReportProto.newBuilder()
+ .addStorageReport(storageReport).build();
+
+ nodeManager.processNodeReport(registeredDatanode, nodeReport);
+ Assert.assertEquals("NumNodeReportProcessed", nrProcessed + 1,
+ getCounter("NumNodeReportProcessed"));
+ }
+
+ /**
+ * Verifies node report processing failure count.
+ */
+ @Test
+ public void testNodeReportProcessingFailure() {
+
+ long nrProcessed = getCounter("NumNodeReportProcessingFailed");
+ DatanodeDetails randomDatanode =
+ MockDatanodeDetails.randomDatanodeDetails();
+
+ StorageReportProto storageReport = TestUtils.createStorageReport(
+ randomDatanode.getUuid(), "/tmp", 100, 10, 90, null);
+
+ NodeReportProto nodeReport = NodeReportProto.newBuilder()
+ .addStorageReport(storageReport).build();
+
+ nodeManager.processNodeReport(randomDatanode, nodeReport);
+ assertEquals("NumNodeReportProcessingFailed", nrProcessed + 1,
+ getCounter("NumNodeReportProcessingFailed"));
+ }
+
+ /**
+ * Verify that datanode aggregated state and capacity metrics are
+ * reported.
+ */
+ @Test
+ public void testNodeCountAndInfoMetricsReported() throws Exception {
+
+ StorageReportProto storageReport = TestUtils.createStorageReport(
+ registeredDatanode.getUuid(), "/tmp", 100, 10, 90, null);
+ NodeReportProto nodeReport = NodeReportProto.newBuilder()
+ .addStorageReport(storageReport).build();
+
+ nodeManager.processNodeReport(registeredDatanode, nodeReport);
+
+ MetricsRecordBuilder metricsSource = getMetrics(SCMNodeMetrics.SOURCE_NAME);
+
+ assertGauge("HealthyNodes", 1, metricsSource);
+ assertGauge("StaleNodes", 0, metricsSource);
+ assertGauge("DeadNodes", 0, metricsSource);
+ assertGauge("DecommissioningNodes", 0, metricsSource);
+ assertGauge("DecommissionedNodes", 0, metricsSource);
+ assertGauge("DiskCapacity", 100L, metricsSource);
+ assertGauge("DiskUsed", 10L, metricsSource);
+ assertGauge("DiskRemaining", 90L, metricsSource);
+ assertGauge("SSDCapacity", 0L, metricsSource);
+ assertGauge("SSDUsed", 0L, metricsSource);
+ assertGauge("SSDRemaining", 0L, metricsSource);
+
+ }
+
+ private long getCounter(String metricName) {
+ return getLongCounter(metricName, getMetrics(SCMNodeMetrics.SOURCE_NAME));
+ }
+
+ private static NodeReportProto createNodeReport() {
+ return NodeReportProto.newBuilder()
+ .addStorageReport(
+ StorageReportProto.newBuilder()
+ .setCapacity(1)
+ .setStorageUuid(UUID.randomUUID().toString())
+ .setStorageLocation("/tmp")
+ .build())
+ .build();
+ }
+}
diff --git a/hadoop-hdds/test-utils/pom.xml b/hadoop-hdds/test-utils/pom.xml
new file mode 100644
index 0000000..814584d
--- /dev/null
+++ b/hadoop-hdds/test-utils/pom.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds</artifactId>
+ <version>0.6.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>hadoop-hdds-test-utils</artifactId>
+ <version>0.6.0-SNAPSHOT</version>
+ <description>Apache Hadoop Distributed Data Store Test Utils</description>
+ <name>Apache Hadoop HDDS Test Utils</name>
+ <packaging>jar</packaging>
+
+ <properties>
+
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.github.spotbugs</groupId>
+ <artifactId>spotbugs-annotations</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/hadoop-hdds/test-utils/src/main/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-hdds/test-utils/src/main/java/org/apache/hadoop/test/GenericTestUtils.java
new file mode 100644
index 0000000..007ecaf
--- /dev/null
+++ b/hadoop-hdds/test-utils/src/main/java/org/apache/hadoop/test/GenericTestUtils.java
@@ -0,0 +1,365 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.io.UnsupportedEncodingException;
+import java.util.concurrent.TimeoutException;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Supplier;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.impl.Log4JLogger;
+import org.apache.log4j.Appender;
+import org.apache.log4j.Layout;
+import org.apache.log4j.Level;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+import org.apache.log4j.PatternLayout;
+import org.apache.log4j.WriterAppender;
+import org.junit.Assert;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Provides some very generic helpers which might be used across the tests.
+ */
+public abstract class GenericTestUtils {
+
+ public static final String SYSPROP_TEST_DATA_DIR = "test.build.data";
+ public static final String DEFAULT_TEST_DATA_DIR;
+ public static final String DEFAULT_TEST_DATA_PATH = "target/test/data/";
+ /**
+ * Error string used in {@link GenericTestUtils#waitFor(Supplier, int, int)}.
+ */
+ public static final String ERROR_MISSING_ARGUMENT =
+ "Input supplier interface should be initialized";
+ public static final String ERROR_INVALID_ARGUMENT =
+ "Total wait time should be greater than check interval time";
+
+ public static final boolean WINDOWS =
+ System.getProperty("os.name").startsWith("Windows");
+
+ private static final long NANOSECONDS_PER_MILLISECOND = 1_000_000;
+
+ static {
+ DEFAULT_TEST_DATA_DIR =
+ "target" + File.separator + "test" + File.separator + "data";
+ }
+
+ /**
+ * Get the (created) base directory for tests.
+ *
+ * @return the absolute directory
+ */
+ public static File getTestDir() {
+ String prop =
+ System.getProperty(SYSPROP_TEST_DATA_DIR, DEFAULT_TEST_DATA_DIR);
+ if (prop.isEmpty()) {
+ // corner case: property is there but empty
+ prop = DEFAULT_TEST_DATA_DIR;
+ }
+ File dir = new File(prop).getAbsoluteFile();
+ assertDirCreation(dir);
+ return dir;
+ }
+
+ /**
+ * Get an uncreated directory for tests.
+ *
+ * @return the absolute directory for tests. Caller is expected to create it.
+ */
+ public static File getTestDir(String subdir) {
+ return new File(getTestDir(), subdir).getAbsoluteFile();
+ }
+
+ /**
+ * Get an uncreated directory for tests with a randomized alphanumeric
+ * name. This is likely to provide a unique path for tests run in parallel
+ *
+ * @return the absolute directory for tests. Caller is expected to create it.
+ */
+ public static File getRandomizedTestDir() {
+ return new File(getRandomizedTempPath());
+ }
+
+ /**
+ * Get a temp path. This may or may not be relative; it depends on what the
+ * {@link #SYSPROP_TEST_DATA_DIR} is set to. If unset, it returns a path
+ * under the relative path {@link #DEFAULT_TEST_DATA_PATH}
+ *
+ * @param subpath sub path, with no leading "/" character
+ * @return a string to use in paths
+ */
+ public static String getTempPath(String subpath) {
+ String prop = WINDOWS ? DEFAULT_TEST_DATA_PATH
+ : System.getProperty(SYSPROP_TEST_DATA_DIR, DEFAULT_TEST_DATA_PATH);
+
+ if (prop.isEmpty()) {
+ // corner case: property is there but empty
+ prop = DEFAULT_TEST_DATA_PATH;
+ }
+ if (!prop.endsWith("/")) {
+ prop = prop + "/";
+ }
+ return prop + subpath;
+ }
+
+ /**
+ * Get a temp path. This may or may not be relative; it depends on what the
+ * {@link #SYSPROP_TEST_DATA_DIR} is set to. If unset, it returns a path
+ * under the relative path {@link #DEFAULT_TEST_DATA_PATH}
+ *
+ * @return a string to use in paths
+ */
+ public static String getRandomizedTempPath() {
+ return getTempPath(RandomStringUtils.randomAlphanumeric(10));
+ }
+
+ /**
+ * Assert that a given file exists.
+ */
+ public static void assertExists(File f) {
+ assertTrue("File " + f + " should exist", f.exists());
+ }
+
+ /**
+ * Assert that a given dir can be created or it already exists.
+ */
+ public static void assertDirCreation(File f) {
+ assertTrue("Could not create dir " + f + ", nor does it exist",
+ f.mkdirs() || f.exists());
+ }
+
+ public static void assertExceptionContains(String expectedText, Throwable t) {
+ assertExceptionContains(expectedText, t, "");
+ }
+
+ public static void assertExceptionContains(String expectedText, Throwable t,
+ String message) {
+ Assert.assertNotNull("Null Throwable", t);
+ String msg = t.toString();
+ if (msg == null) {
+ throw new AssertionError("Null Throwable.toString() value", t);
+ } else if (expectedText != null && !msg.contains(expectedText)) {
+ String prefix = StringUtils.isEmpty(message) ? "" : message + ": ";
+ throw new AssertionError(String
+ .format("%s Expected to find '%s' %s: %s", prefix, expectedText,
+ "but got unexpected exception",
+ stringifyException(t)), t);
+ }
+ }
+
+ /**
+ * Make a string representation of the exception.
+ * @param e The exception to stringify
+ * @return A string with exception name and call stack.
+ */
+ public static String stringifyException(Throwable e) {
+ StringWriter stm = new StringWriter();
+ PrintWriter wrt = new PrintWriter(stm);
+ e.printStackTrace(wrt);
+ wrt.close();
+ return stm.toString();
+ }
+
+ /**
+ * Wait for the specified test to return true. The test will be performed
+ * initially and then every {@code checkEveryMillis} until at least
+ * {@code waitForMillis} time has expired. If {@code check} is null or
+ * {@code waitForMillis} is less than {@code checkEveryMillis} this method
+ * will throw an {@link IllegalArgumentException}.
+ *
+ * @param check the test to perform
+ * @param checkEveryMillis how often to perform the test
+ * @param waitForMillis the amount of time after which no more tests
+ * will be
+ * performed
+ * @throws TimeoutException if the test does not return true in the
+ * allotted
+ * time
+ * @throws InterruptedException if the method is interrupted while waiting
+ */
+ public static void waitFor(Supplier<Boolean> check, int checkEveryMillis,
+ int waitForMillis) throws TimeoutException, InterruptedException {
+ Preconditions.checkNotNull(check, ERROR_MISSING_ARGUMENT);
+ Preconditions.checkArgument(waitForMillis >= checkEveryMillis,
+ ERROR_INVALID_ARGUMENT);
+
+ long st = monotonicNow();
+ boolean result = check.get();
+
+ while (!result && (monotonicNow() - st < waitForMillis)) {
+ Thread.sleep(checkEveryMillis);
+ result = check.get();
+ }
+
+ if (!result) {
+ throw new TimeoutException("Timed out waiting for condition. " +
+ "Thread diagnostics:\n" +
+ TimedOutTestsListener.buildThreadDiagnosticString());
+ }
+ }
+
+ /**
+ * @deprecated use sl4fj based version
+ */
+ @Deprecated
+ public static void setLogLevel(Logger logger, Level level) {
+ logger.setLevel(level);
+ }
+
+ public static void setLogLevel(org.slf4j.Logger logger,
+ org.slf4j.event.Level level) {
+ setLogLevel(toLog4j(logger), Level.toLevel(level.toString()));
+ }
+
+ public static void setRootLogLevel(org.slf4j.event.Level level) {
+ setLogLevel(LogManager.getRootLogger(), Level.toLevel(level.toString()));
+ }
+
+ /**
+ * Class to capture logs for doing assertions.
+ */
+ public static final class LogCapturer {
+ private StringWriter sw = new StringWriter();
+ private WriterAppender appender;
+ private Logger logger;
+
+ public static LogCapturer captureLogs(Log l) {
+ Logger logger = ((Log4JLogger) l).getLogger();
+ return new LogCapturer(logger);
+ }
+
+ public static LogCapturer captureLogs(org.slf4j.Logger logger) {
+ return new LogCapturer(toLog4j(logger));
+ }
+
+ private LogCapturer(Logger logger) {
+ this.logger = logger;
+ Appender defaultAppender = Logger.getRootLogger().getAppender("stdout");
+ if (defaultAppender == null) {
+ defaultAppender = Logger.getRootLogger().getAppender("console");
+ }
+ final Layout layout = (defaultAppender == null) ? new PatternLayout() :
+ defaultAppender.getLayout();
+ this.appender = new WriterAppender(layout, sw);
+ logger.addAppender(this.appender);
+ }
+
+ public String getOutput() {
+ return sw.toString();
+ }
+
+ public void stopCapturing() {
+ logger.removeAppender(appender);
+ }
+
+ public void clearOutput() {
+ sw.getBuffer().setLength(0);
+ }
+ }
+
+ @Deprecated
+ public static Logger toLog4j(org.slf4j.Logger logger) {
+ return LogManager.getLogger(logger.getName());
+ }
+
+ private static long monotonicNow() {
+ return System.nanoTime() / NANOSECONDS_PER_MILLISECOND;
+ }
+
+ /**
+ * Capture output printed to {@link System#err}.
+ * <p>
+ * Usage:
+ * <pre>
+ * try (SystemErrCapturer capture = new SystemErrCapturer()) {
+ * ...
+ * // Call capture.getOutput() to get the output string
+ * }
+ * </pre>
+ * <p>
+ * TODO: Add lambda support once Java 8 is common.
+ * <pre>
+ * SystemErrCapturer.withCapture(capture -> {
+ * ...
+ * })
+ * </pre>
+ */
+ public static class SystemErrCapturer implements AutoCloseable {
+ private final ByteArrayOutputStream bytes;
+ private final PrintStream bytesPrintStream;
+ private final PrintStream oldErr;
+
+ public SystemErrCapturer() throws UnsupportedEncodingException {
+ bytes = new ByteArrayOutputStream();
+ bytesPrintStream = new PrintStream(bytes, false, UTF_8.name());
+ oldErr = System.err;
+ System.setErr(new TeePrintStream(oldErr, bytesPrintStream));
+ }
+
+ public String getOutput() throws UnsupportedEncodingException {
+ return bytes.toString(UTF_8.name());
+ }
+
+ @Override
+ public void close() throws Exception {
+ IOUtils.closeQuietly(bytesPrintStream);
+ System.setErr(oldErr);
+ }
+ }
+
+ /**
+ * Prints output to one {@link PrintStream} while copying to the other.
+ * <p>
+ * Closing the main {@link PrintStream} will NOT close the other.
+ */
+ public static class TeePrintStream extends PrintStream {
+ private final PrintStream other;
+
+ public TeePrintStream(OutputStream main, PrintStream other)
+ throws UnsupportedEncodingException {
+ super(main, false, UTF_8.name());
+ this.other = other;
+ }
+
+ @Override
+ public void flush() {
+ super.flush();
+ other.flush();
+ }
+
+ @Override
+ public void write(byte[] buf, int off, int len) {
+ super.write(buf, off, len);
+ other.write(buf, off, len);
+ }
+ }
+
+}
diff --git a/hadoop-hdds/test-utils/src/main/java/org/apache/hadoop/test/LambdaTestUtils.java b/hadoop-hdds/test-utils/src/main/java/org/apache/hadoop/test/LambdaTestUtils.java
new file mode 100644
index 0000000..6689dee
--- /dev/null
+++ b/hadoop-hdds/test-utils/src/main/java/org/apache/hadoop/test/LambdaTestUtils.java
@@ -0,0 +1,814 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.test;
+
+import java.util.Optional;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeoutException;
+
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+
+import com.google.common.base.Preconditions;
+import org.junit.Assert;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Class to make the most of Lambda expressions in Ozone tests.
+ *
+ * The code has been designed from the outset to be Java-8 friendly, but
+ * to still be usable in Java 7.
+ *
+ * The code is modelled on {@code GenericTestUtils#waitFor(Supplier, int, int)},
+ * but also lifts concepts from Scalatest's {@code awaitResult} and
+ * its notion of pluggable retry logic (simple, backoff, maybe even things
+ * with jitter: test author gets to choose).
+ * The {@link #intercept(Class, Callable)} method is also all credit due
+ * Scalatest, though it's been extended to also support a string message
+ * check; useful when checking the contents of the exception.
+ */
+public final class LambdaTestUtils {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(LambdaTestUtils.class);
+
+ private LambdaTestUtils() {
+ }
+
+ /**
+ * This is the string included in the assertion text in
+ * {@link #intercept(Class, Callable)} if
+ * the closure returned a null value.
+ */
+ public static final String NULL_RESULT = "(null)";
+
+ /**
+ * Interface to implement for converting a timeout into some form
+ * of exception to raise.
+ */
+ @FunctionalInterface
+ public interface TimeoutHandler {
+
+ /**
+ * Create an exception (or throw one, if desired).
+ * @param timeoutMillis timeout which has arisen
+ * @param caught any exception which was caught; may be null
+ * @return an exception which will then be thrown
+ * @throws Exception if the handler wishes to raise an exception
+ * that way.
+ */
+ Throwable evaluate(int timeoutMillis, Throwable caught) throws Throwable;
+ }
+
+ /**
+ * Wait for a condition to be met, with a retry policy returning the
+ * sleep time before the next attempt is made. If, at the end
+ * of the timeout period, the condition is still false (or failing with
+ * an exception), the timeout handler is invoked, passing in the timeout
+ * and any exception raised in the last invocation. The exception returned
+ * by this timeout handler is then rethrown.
+ * <p>
+ * Example: Wait 30s for a condition to be met, with a sleep of 30s
+ * between each probe.
+ * If the operation is failing, then, after 30s, the timeout handler
+ * is called. This returns the exception passed in (if any),
+ * or generates a new one.
+ * <pre>
+ * await(
+ * 30 * 1000,
+ * () -> { return 0 == filesystem.listFiles(new Path("/")).length); },
+ * () -> 500),
+ * (timeout, ex) -> ex != null ? ex : new TimeoutException("timeout"));
+ * </pre>
+ *
+ * @param timeoutMillis timeout in milliseconds.
+ * Can be zero, in which case only one attempt is made.
+ * @param check predicate to evaluate
+ * @param retry retry escalation logic
+ * @param timeoutHandler handler invoked on timeout;
+ * the returned exception will be thrown
+ * @return the number of iterations before the condition was satisfied
+ * @throws Exception the exception returned by {@code timeoutHandler} on
+ * timeout
+ * @throws FailFastException immediately if the evaluated operation raises it
+ * @throws InterruptedException if interrupted.
+ */
+ public static int await(int timeoutMillis,
+ Callable<Boolean> check,
+ Callable<Integer> retry,
+ TimeoutHandler timeoutHandler)
+ throws Exception {
+ Preconditions.checkArgument(timeoutMillis >= 0,
+ "timeoutMillis must be >= 0");
+ Preconditions.checkNotNull(timeoutHandler);
+
+ final long endTime = System.currentTimeMillis() + timeoutMillis;
+ Throwable ex = null;
+ boolean running = true;
+ int iterations = 0;
+ while (running) {
+ iterations++;
+ try {
+ if (check.call()) {
+ return iterations;
+ }
+ // the probe failed but did not raise an exception. Reset any
+ // exception raised by a previous probe failure.
+ ex = null;
+ } catch (InterruptedException
+ | FailFastException
+ | VirtualMachineError e) {
+ throw e;
+ } catch (Throwable e) {
+ LOG.debug("eventually() iteration {}", iterations, e);
+ ex = e;
+ }
+ running = System.currentTimeMillis() < endTime;
+ if (running) {
+ int sleeptime = retry.call();
+ if (sleeptime >= 0) {
+ Thread.sleep(sleeptime);
+ } else {
+ running = false;
+ }
+ }
+ }
+ // timeout
+ Throwable evaluate;
+ try {
+ evaluate = timeoutHandler.evaluate(timeoutMillis, ex);
+ if (evaluate == null) {
+ // bad timeout handler logic; fall back to GenerateTimeout so the
+ // underlying problem isn't lost.
+ LOG.error("timeout handler {} did not throw an exception ",
+ timeoutHandler);
+ evaluate = new GenerateTimeout().evaluate(timeoutMillis, ex);
+ }
+ } catch (Throwable throwable) {
+ evaluate = throwable;
+ }
+ return raise(evaluate);
+ }
+
+ /**
+ * Simplified {@link #await(int, Callable, Callable, TimeoutHandler)}
+ * operation with a fixed interval
+ * and {@link GenerateTimeout} handler to generate a {@code TimeoutException}.
+ * <p>
+ * Example: await for probe to succeed:
+ * <pre>
+ * await(
+ * 30 * 1000, 500,
+ * () -> { return 0 == filesystem.listFiles(new Path("/")).length); });
+ * </pre>
+ *
+ * @param timeoutMillis timeout in milliseconds.
+ * Can be zero, in which case only one attempt is made.
+ * @param intervalMillis interval in milliseconds between checks
+ * @param check predicate to evaluate
+ * @return the number of iterations before the condition was satisfied
+ * @throws Exception returned by {@code failure} on timeout
+ * @throws FailFastException immediately if the evaluated operation raises it
+ * @throws InterruptedException if interrupted.
+ */
+ public static int await(int timeoutMillis,
+ int intervalMillis,
+ Callable<Boolean> check) throws Exception {
+ return await(timeoutMillis, check,
+ new FixedRetryInterval(intervalMillis),
+ new GenerateTimeout());
+ }
+
+ /**
+ * Repeatedly execute a closure until it returns a value rather than
+ * raise an exception.
+ * Exceptions are caught and, with one exception,
+ * trigger a sleep and retry. This is similar of ScalaTest's
+ * {@code eventually(timeout, closure)} operation, though that lacks
+ * the ability to fail fast if the inner closure has determined that
+ * a failure condition is non-recoverable.
+ * <p>
+ * Example: spin until an the number of files in a filesystem is non-zero,
+ * returning the files found.
+ * The sleep interval backs off by 500 ms each iteration to a maximum of 5s.
+ * <pre>
+ * FileStatus[] files = eventually( 30 * 1000,
+ * () -> {
+ * FileStatus[] f = filesystem.listFiles(new Path("/"));
+ * assertEquals(0, f.length);
+ * return f;
+ * },
+ * new ProportionalRetryInterval(500, 5000));
+ * </pre>
+ * This allows for a fast exit, yet reduces probe frequency over time.
+ *
+ * @param <T> return type
+ * @param timeoutMillis timeout in milliseconds.
+ * Can be zero, in which case only one attempt is made before failing.
+ * @param eval expression to evaluate
+ * @param retry retry interval generator
+ * @return result of the first successful eval call
+ * @throws Exception the last exception thrown before timeout was triggered
+ * @throws FailFastException if raised -without any retry attempt.
+ * @throws InterruptedException if interrupted during the sleep operation.
+ * @throws OutOfMemoryError you've run out of memory.
+ */
+ @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
+ public static <T> T eventually(int timeoutMillis,
+ Callable<T> eval,
+ Callable<Integer> retry) throws Exception {
+ Preconditions.checkArgument(timeoutMillis >= 0,
+ "timeoutMillis must be >= 0");
+ final long endTime = System.currentTimeMillis() + timeoutMillis;
+ Throwable ex;
+ boolean running;
+ int iterations = 0;
+ do {
+ iterations++;
+ try {
+ return eval.call();
+ } catch (InterruptedException
+ | FailFastException
+ | VirtualMachineError e) {
+ // these two exceptions trigger an immediate exit
+ throw e;
+ } catch (Throwable e) {
+ LOG.debug("evaluate() iteration {}", iterations, e);
+ ex = e;
+ running = System.currentTimeMillis() < endTime;
+ int sleeptime = retry.call();
+ if (running && sleeptime >= 0) {
+ Thread.sleep(sleeptime);
+ }
+ }
+ } while (running);
+ // timeout. Throw the last exception raised
+ return raise(ex);
+ }
+
+ /**
+ * Take the throwable and raise it as an exception or an error, depending
+ * upon its type. This allows callers to declare that they only throw
+ * Exception (i.e. can be invoked by Callable) yet still rethrow a
+ * previously caught Throwable.
+ * @param throwable Throwable to rethrow
+ * @param <T> expected return type
+ * @return never
+ * @throws Exception if throwable is an Exception
+ * @throws Error if throwable is not an Exception
+ */
+ private static <T> T raise(Throwable throwable) throws Exception {
+ if (throwable instanceof Exception) {
+ throw (Exception) throwable;
+ } else {
+ throw (Error) throwable;
+ }
+ }
+
+ /**
+ * Variant of {@link #eventually(int, Callable, Callable)} method for
+ * void lambda expressions.
+ * @param timeoutMillis timeout in milliseconds.
+ * Can be zero, in which case only one attempt is made before failing.
+ * @param eval expression to evaluate
+ * @param retry retry interval generator
+ * @throws Exception the last exception thrown before timeout was triggered
+ * @throws FailFastException if raised -without any retry attempt.
+ * @throws InterruptedException if interrupted during the sleep operation.
+ */
+ public static void eventually(int timeoutMillis,
+ VoidCallable eval,
+ Callable<Integer> retry) throws Exception {
+ eventually(timeoutMillis, new VoidCaller(eval), retry);
+ }
+
+ /**
+ * Simplified {@link #eventually(int, Callable, Callable)} method
+ * with a fixed interval.
+ * <p>
+ * Example: wait 30s until an assertion holds, sleeping 1s between each
+ * check.
+ * <pre>
+ * eventually( 30 * 1000, 1000,
+ * () -> { assertEquals(0, filesystem.listFiles(new Path("/")).length); }
+ * );
+ * </pre>
+ *
+ * @param timeoutMillis timeout in milliseconds.
+ * Can be zero, in which case only one attempt is made before failing.
+ * @param intervalMillis interval in milliseconds
+ * @param eval expression to evaluate
+ * @return result of the first successful invocation of {@code eval()}
+ * @throws Exception the last exception thrown before timeout was triggered
+ * @throws FailFastException if raised -without any retry attempt.
+ * @throws InterruptedException if interrupted during the sleep operation.
+ */
+ public static <T> T eventually(int timeoutMillis,
+ int intervalMillis,
+ Callable<T> eval) throws Exception {
+ return eventually(timeoutMillis, eval,
+ new FixedRetryInterval(intervalMillis));
+ }
+
+ /**
+ /**
+ * Variant of {@link #eventually(int, int, Callable)} method for
+ * void lambda expressions.
+ * @param timeoutMillis timeout in milliseconds.
+ * Can be zero, in which case only one attempt is made before failing.
+ * @param intervalMillis interval in milliseconds
+ * @param eval expression to evaluate
+ * @throws Exception the last exception thrown before timeout was triggered
+ * @throws FailFastException if raised -without any retry attempt.
+ * @throws InterruptedException if interrupted during the sleep operation.
+ */
+ public static void eventually(int timeoutMillis,
+ int intervalMillis,
+ VoidCallable eval) throws Exception {
+ eventually(timeoutMillis, eval,
+ new FixedRetryInterval(intervalMillis));
+ }
+
+ /**
+ * Intercept an exception; throw an {@code AssertionError} if one not raised.
+ * The caught exception is rethrown if it is of the wrong class or
+ * does not contain the text defined in {@code contained}.
+ * <p>
+ * Example: expect deleting a nonexistent file to raise a
+ * {@code FileNotFoundException}.
+ * <pre>
+ * FileNotFoundException ioe = intercept(FileNotFoundException.class,
+ * () -> {
+ * filesystem.delete(new Path("/missing"), false);
+ * });
+ * </pre>
+ *
+ * @param clazz class of exception; the raised exception must be this class
+ * <i>or a subclass</i>.
+ * @param eval expression to eval
+ * @param <T> return type of expression
+ * @param <E> exception class
+ * @return the caught exception if it was of the expected type
+ * @throws Exception any other exception raised
+ * @throws AssertionError if the evaluation call didn't raise an exception.
+ * The error includes the {@code toString()} value of the result, if this
+ * can be determined.
+ */
+ @SuppressWarnings("unchecked")
+ public static <T, E extends Throwable> E intercept(
+ Class<E> clazz,
+ Callable<T> eval)
+ throws Exception {
+ return intercept(clazz,
+ null,
+ "Expected a " + clazz.getName() + " to be thrown," +
+ " but got the result: ",
+ eval);
+ }
+
+ /**
+ * Variant of {@link #intercept(Class, Callable)} to simplify void
+ * invocations.
+ * @param clazz class of exception; the raised exception must be this class
+ * <i>or a subclass</i>.
+ * @param eval expression to eval
+ * @param <E> exception class
+ * @return the caught exception if it was of the expected type
+ * @throws Exception any other exception raised
+ * @throws AssertionError if the evaluation call didn't raise an exception.
+ */
+ @SuppressWarnings("unchecked")
+ public static <E extends Throwable> E intercept(
+ Class<E> clazz,
+ VoidCallable eval)
+ throws Exception {
+ try {
+ eval.call();
+ throw new AssertionError("Expected an exception");
+ } catch (Throwable e) {
+ if (clazz.isAssignableFrom(e.getClass())) {
+ return (E)e;
+ }
+ throw e;
+ }
+ }
+
+ /**
+ * Intercept an exception; throw an {@code AssertionError} if one not raised.
+ * The caught exception is rethrown if it is of the wrong class or
+ * does not contain the text defined in {@code contained}.
+ * <p>
+ * Example: expect deleting a nonexistent file to raise a
+ * {@code FileNotFoundException} with the {@code toString()} value
+ * containing the text {@code "missing"}.
+ * <pre>
+ * FileNotFoundException ioe = intercept(FileNotFoundException.class,
+ * "missing",
+ * () -> {
+ * filesystem.delete(new Path("/missing"), false);
+ * });
+ * </pre>
+ *
+ * @param clazz class of exception; the raised exception must be this class
+ * <i>or a subclass</i>.
+ * @param contained string which must be in the {@code toString()} value
+ * of the exception
+ * @param eval expression to eval
+ * @param <T> return type of expression
+ * @param <E> exception class
+ * @return the caught exception if it was of the expected type and contents
+ * @throws Exception any other exception raised
+ * @throws AssertionError if the evaluation call didn't raise an exception.
+ * The error includes the {@code toString()} value of the result, if this
+ * can be determined.
+ * @see GenericTestUtils#assertExceptionContains(String, Throwable)
+ */
+ public static <T, E extends Throwable> E intercept(
+ Class<E> clazz,
+ String contained,
+ Callable<T> eval)
+ throws Exception {
+ E ex = intercept(clazz, eval);
+ GenericTestUtils.assertExceptionContains(contained, ex);
+ return ex;
+ }
+
+ /**
+ * Intercept an exception; throw an {@code AssertionError} if one not raised.
+ * The caught exception is rethrown if it is of the wrong class or
+ * does not contain the text defined in {@code contained}.
+ * <p>
+ * Example: expect deleting a nonexistent file to raise a
+ * {@code FileNotFoundException} with the {@code toString()} value
+ * containing the text {@code "missing"}.
+ * <pre>
+ * FileNotFoundException ioe = intercept(FileNotFoundException.class,
+ * "missing",
+ * "path should not be found",
+ * () -> {
+ * filesystem.delete(new Path("/missing"), false);
+ * });
+ * </pre>
+ *
+ * @param clazz class of exception; the raised exception must be this class
+ * <i>or a subclass</i>.
+ * @param contained string which must be in the {@code toString()} value
+ * of the exception
+ * @param message any message tho include in exception/log messages
+ * @param eval expression to eval
+ * @param <T> return type of expression
+ * @param <E> exception class
+ * @return the caught exception if it was of the expected type and contents
+ * @throws Exception any other exception raised
+ * @throws AssertionError if the evaluation call didn't raise an exception.
+ * The error includes the {@code toString()} value of the result, if this
+ * can be determined.
+ * @see GenericTestUtils#assertExceptionContains(String, Throwable)
+ */
+ public static <T, E extends Throwable> E intercept(
+ Class<E> clazz,
+ String contained,
+ String message,
+ Callable<T> eval)
+ throws Exception {
+ E ex;
+ try {
+ T result = eval.call();
+ throw new AssertionError(message + ": " + robustToString(result));
+ } catch (Throwable e) {
+ if (!clazz.isAssignableFrom(e.getClass())) {
+ throw e;
+ } else {
+ ex = (E) e;
+ }
+ }
+ GenericTestUtils.assertExceptionContains(contained, ex, message);
+ return ex;
+ }
+
+ /**
+ * Variant of {@link #intercept(Class, Callable)} to simplify void
+ * invocations.
+ * @param clazz class of exception; the raised exception must be this class
+ * <i>or a subclass</i>.
+ * @param contained string which must be in the {@code toString()} value
+ * of the exception
+ * @param eval expression to eval
+ * @param <E> exception class
+ * @return the caught exception if it was of the expected type
+ * @throws Exception any other exception raised
+ * @throws AssertionError if the evaluation call didn't raise an exception.
+ */
+ public static <E extends Throwable> E intercept(
+ Class<E> clazz,
+ String contained,
+ VoidCallable eval)
+ throws Exception {
+ return intercept(clazz, contained,
+ "Expecting " + clazz.getName()
+ + (contained != null? (" with text " + contained) : "")
+ + " but got ",
+ () -> {
+ eval.call();
+ return "void";
+ });
+ }
+
+ /**
+ * Variant of {@link #intercept(Class, Callable)} to simplify void
+ * invocations.
+ * @param clazz class of exception; the raised exception must be this class
+ * <i>or a subclass</i>.
+ * @param contained string which must be in the {@code toString()} value
+ * of the exception
+ * @param message any message tho include in exception/log messages
+ * @param eval expression to eval
+ * @param <E> exception class
+ * @return the caught exception if it was of the expected type
+ * @throws Exception any other exception raised
+ * @throws AssertionError if the evaluation call didn't raise an exception.
+ */
+ public static <E extends Throwable> E intercept(
+ Class<E> clazz,
+ String contained,
+ String message,
+ VoidCallable eval)
+ throws Exception {
+ return intercept(clazz, contained, message,
+ () -> {
+ eval.call();
+ return "void";
+ });
+ }
+
+ /**
+ * Robust string converter for exception messages; if the {@code toString()}
+ * method throws an exception then that exception is caught and logged,
+ * then a simple string of the classname logged.
+ * This stops a {@code toString()} failure hiding underlying problems.
+ * @param o object to stringify
+ * @return a string for exception messages
+ */
+ private static String robustToString(Object o) {
+ if (o == null) {
+ return NULL_RESULT;
+ } else {
+ try {
+ return o.toString();
+ } catch (Exception e) {
+ LOG.info("Exception calling toString()", e);
+ return o.getClass().toString();
+ }
+ }
+ }
+
+ /**
+ * Assert that an optional value matches an expected one;
+ * checks include null and empty on the actual value.
+ * @param message message text
+ * @param expected expected value
+ * @param actual actual optional value
+ * @param <T> type
+ */
+ public static <T> void assertOptionalEquals(String message,
+ T expected,
+ Optional<T> actual) {
+ Assert.assertNotNull(message, actual);
+ Assert.assertTrue(message +" -not present", actual.isPresent());
+ Assert.assertEquals(message, expected, actual.get());
+ }
+
+ /**
+ * Assert that an optional value matches an expected one;
+ * checks include null and empty on the actual value.
+ * @param message message text
+ * @param actual actual optional value
+ * @param <T> type
+ */
+ public static <T> void assertOptionalUnset(String message,
+ Optional<T> actual) {
+ Assert.assertNotNull(message, actual);
+ actual.ifPresent(
+ t -> Assert.fail("Expected empty option, got " + t.toString()));
+ }
+
+ /**
+ * Invoke a callable; wrap all checked exceptions with an
+ * AssertionError.
+ * @param closure closure to execute
+ * @param <T> return type of closure
+ * @return the value of the closure
+ * @throws AssertionError if the operation raised an IOE or
+ * other checked exception.
+ */
+ public static <T> T eval(Callable<T> closure) {
+ try {
+ return closure.call();
+ } catch (RuntimeException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AssertionError(e.toString(), e);
+ }
+ }
+
+ /**
+ * Invoke a callable; wrap all checked exceptions with an
+ * AssertionError.
+ * @param closure closure to execute
+ * @return the value of the closure
+ * @throws AssertionError if the operation raised an IOE or
+ * other checked exception.
+ */
+ public static void eval(VoidCallable closure) {
+ try {
+ closure.call();
+ } catch (RuntimeException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AssertionError(e.toString(), e);
+ }
+ }
+
+ /**
+ * Returns {@code TimeoutException} on a timeout. If
+ * there was a inner class passed in, includes it as the
+ * inner failure.
+ */
+ public static class GenerateTimeout implements TimeoutHandler {
+ private final String message;
+
+ public GenerateTimeout(String message) {
+ this.message = message;
+ }
+
+ public GenerateTimeout() {
+ this("timeout");
+ }
+
+ /**
+ * Evaluate operation creates a new {@code TimeoutException}.
+ * @param timeoutMillis timeout in millis
+ * @param caught optional caught exception
+ * @return TimeoutException
+ */
+ @Override
+ public Throwable evaluate(int timeoutMillis, Throwable caught)
+ throws Throwable {
+ String s = String.format("%s: after %d millis", message,
+ timeoutMillis);
+ String caughtText = caught != null
+ ? ("; " + robustToString(caught)) : "";
+
+ return (new TimeoutException(s + caughtText)
+ .initCause(caught));
+ }
+ }
+
+ /**
+ * Retry at a fixed time period between calls.
+ */
+ public static class FixedRetryInterval implements Callable<Integer> {
+ private final int intervalMillis;
+ private int invocationCount = 0;
+
+ public FixedRetryInterval(int intervalMillis) {
+ Preconditions.checkArgument(intervalMillis > 0);
+ this.intervalMillis = intervalMillis;
+ }
+
+ @Override
+ public Integer call() throws Exception {
+ invocationCount++;
+ return intervalMillis;
+ }
+
+ public int getInvocationCount() {
+ return invocationCount;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "FixedRetryInterval{");
+ sb.append("interval=").append(intervalMillis);
+ sb.append(", invocationCount=").append(invocationCount);
+ sb.append('}');
+ return sb.toString();
+ }
+ }
+
+ /**
+ * Gradually increase the sleep time by the initial interval, until
+ * the limit set by {@code maxIntervalMillis} is reached.
+ */
+ public static class ProportionalRetryInterval implements Callable<Integer> {
+ private final int intervalMillis;
+ private final int maxIntervalMillis;
+ private int current;
+ private int invocationCount = 0;
+
+ public ProportionalRetryInterval(int intervalMillis,
+ int maxIntervalMillis) {
+ Preconditions.checkArgument(intervalMillis > 0);
+ Preconditions.checkArgument(maxIntervalMillis > 0);
+ this.intervalMillis = intervalMillis;
+ this.current = intervalMillis;
+ this.maxIntervalMillis = maxIntervalMillis;
+ }
+
+ @Override
+ public Integer call() throws Exception {
+ invocationCount++;
+ int last = current;
+ if (last < maxIntervalMillis) {
+ current += intervalMillis;
+ }
+ return last;
+ }
+
+ public int getInvocationCount() {
+ return invocationCount;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "ProportionalRetryInterval{");
+ sb.append("interval=").append(intervalMillis);
+ sb.append(", current=").append(current);
+ sb.append(", limit=").append(maxIntervalMillis);
+ sb.append(", invocationCount=").append(invocationCount);
+ sb.append('}');
+ return sb.toString();
+ }
+ }
+
+ /**
+ * An exception which triggers a fast exist from the
+ * {@link #eventually(int, Callable, Callable)} and
+ * {@link #await(int, Callable, Callable, TimeoutHandler)} loops.
+ */
+ public static class FailFastException extends Exception {
+
+ public FailFastException(String detailMessage) {
+ super(detailMessage);
+ }
+
+ public FailFastException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ /**
+ * Instantiate from a format string.
+ * @param format format string
+ * @param args arguments to format
+ * @return an instance with the message string constructed.
+ */
+ public static FailFastException newInstance(String format, Object...args) {
+ return new FailFastException(String.format(format, args));
+ }
+ }
+
+ /**
+ * A simple interface for lambdas, which returns nothing; this exists
+ * to simplify lambda tests on operations with no return value.
+ */
+ @FunctionalInterface
+ public interface VoidCallable {
+ void call() throws Exception;
+ }
+
+ /**
+ * Bridge class to make {@link VoidCallable} something to use in anything
+ * which takes an {@link Callable}.
+ */
+ public static class VoidCaller implements Callable<Void> {
+ private final VoidCallable callback;
+
+ public VoidCaller(VoidCallable callback) {
+ this.callback = callback;
+ }
+
+ @Override
+ public Void call() throws Exception {
+ callback.call();
+ return null;
+ }
+ }
+
+}
diff --git a/hadoop-hdds/test-utils/src/main/java/org/apache/hadoop/test/TimedOutTestsListener.java b/hadoop-hdds/test-utils/src/main/java/org/apache/hadoop/test/TimedOutTestsListener.java
new file mode 100644
index 0000000..f111f7d
--- /dev/null
+++ b/hadoop-hdds/test-utils/src/main/java/org/apache/hadoop/test/TimedOutTestsListener.java
@@ -0,0 +1,183 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.test;
+
+import java.io.BufferedWriter;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.lang.management.LockInfo;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MonitorInfo;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Locale;
+import java.util.Map;
+
+import org.junit.runner.notification.Failure;
+import org.junit.runner.notification.RunListener;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+/**
+ * JUnit run listener which prints full thread dump into System.err
+ * in case a test is failed due to timeout.
+ */
+public class TimedOutTestsListener extends RunListener {
+
+ private static final String TEST_TIMED_OUT_PREFIX = "test timed out after";
+
+ private static final String INDENT = " ";
+
+ private final PrintWriter output;
+
+ public TimedOutTestsListener() {
+ this(new PrintWriter(new BufferedWriter(new OutputStreamWriter(
+ System.err, UTF_8))));
+ }
+
+ public TimedOutTestsListener(PrintWriter output) {
+ this.output = output;
+ }
+
+ @Override
+ public void testFailure(Failure failure) throws Exception {
+ if (failure != null && failure.getMessage() != null
+ && failure.getMessage().startsWith(TEST_TIMED_OUT_PREFIX)) {
+ output.println("====> TEST TIMED OUT. PRINTING THREAD DUMP. <====");
+ output.println();
+ output.print(buildThreadDiagnosticString());
+ }
+ }
+
+ public static String buildThreadDiagnosticString() {
+ StringWriter sw = new StringWriter();
+ PrintWriter output = new PrintWriter(sw);
+
+ DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss,SSS");
+ output
+ .println(String.format("Timestamp: %s", dateFormat.format(new Date())));
+ output.println();
+ output.println(buildThreadDump());
+
+ String deadlocksInfo = buildDeadlockInfo();
+ if (deadlocksInfo != null) {
+ output.println("====> DEADLOCKS DETECTED <====");
+ output.println();
+ output.println(deadlocksInfo);
+ }
+
+ return sw.toString();
+ }
+
+ static String buildThreadDump() {
+ StringBuilder dump = new StringBuilder();
+ Map<Thread, StackTraceElement[]> stackTraces = Thread.getAllStackTraces();
+ for (Map.Entry<Thread, StackTraceElement[]> e : stackTraces.entrySet()) {
+ Thread thread = e.getKey();
+ dump.append(String.format(
+ "\"%s\" %s prio=%d tid=%d %s%njava.lang.Thread.State: %s",
+ thread.getName(),
+ (thread.isDaemon() ? "daemon" : ""),
+ thread.getPriority(),
+ thread.getId(),
+ Thread.State.WAITING.equals(thread.getState()) ?
+ "in Object.wait()" :
+ thread.getState().name().toLowerCase(Locale.ENGLISH),
+ Thread.State.WAITING.equals(thread.getState()) ?
+ "WAITING (on object monitor)" : thread.getState()));
+ for (StackTraceElement stackTraceElement : e.getValue()) {
+ dump.append("\n at ");
+ dump.append(stackTraceElement);
+ }
+ dump.append("\n");
+ }
+ return dump.toString();
+ }
+
+ static String buildDeadlockInfo() {
+ ThreadMXBean threadBean = ManagementFactory.getThreadMXBean();
+ long[] threadIds = threadBean.findMonitorDeadlockedThreads();
+ if (threadIds != null && threadIds.length > 0) {
+ StringWriter stringWriter = new StringWriter();
+ PrintWriter out = new PrintWriter(stringWriter);
+
+ ThreadInfo[] infos = threadBean.getThreadInfo(threadIds, true, true);
+ for (ThreadInfo ti : infos) {
+ printThreadInfo(ti, out);
+ printLockInfo(ti.getLockedSynchronizers(), out);
+ out.println();
+ }
+
+ out.close();
+ return stringWriter.toString();
+ } else {
+ return null;
+ }
+ }
+
+ private static void printThreadInfo(ThreadInfo ti, PrintWriter out) {
+ // print thread information
+ printThread(ti, out);
+
+ // print stack trace with locks
+ StackTraceElement[] stacktrace = ti.getStackTrace();
+ MonitorInfo[] monitors = ti.getLockedMonitors();
+ for (int i = 0; i < stacktrace.length; i++) {
+ StackTraceElement ste = stacktrace[i];
+ out.println(INDENT + "at " + ste.toString());
+ for (MonitorInfo mi : monitors) {
+ if (mi.getLockedStackDepth() == i) {
+ out.println(INDENT + " - locked " + mi);
+ }
+ }
+ }
+ out.println();
+ }
+
+ private static void printThread(ThreadInfo ti, PrintWriter out) {
+ out.print("\"" + ti.getThreadName() + "\"" + " Id="
+ + ti.getThreadId() + " in " + ti.getThreadState());
+ if (ti.getLockName() != null) {
+ out.print(" on lock=" + ti.getLockName());
+ }
+ if (ti.isSuspended()) {
+ out.print(" (suspended)");
+ }
+ if (ti.isInNative()) {
+ out.print(" (running in native)");
+ }
+ out.println();
+ if (ti.getLockOwnerName() != null) {
+ out.println(INDENT + " owned by " + ti.getLockOwnerName() + " Id="
+ + ti.getLockOwnerId());
+ }
+ }
+
+ private static void printLockInfo(LockInfo[] locks, PrintWriter out) {
+ out.println(INDENT + "Locked synchronizers: count = " + locks.length);
+ for (LockInfo li : locks) {
+ out.println(INDENT + " - " + li);
+ }
+ out.println();
+ }
+
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/web/utils/package-info.java b/hadoop-hdds/test-utils/src/main/java/org/apache/hadoop/test/package-info.java
similarity index 92%
rename from hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/web/utils/package-info.java
rename to hadoop-hdds/test-utils/src/main/java/org/apache/hadoop/test/package-info.java
index e5812c0..d74e033 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/web/utils/package-info.java
+++ b/hadoop-hdds/test-utils/src/main/java/org/apache/hadoop/test/package-info.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -16,4 +16,7 @@
* limitations under the License.
*/
-package org.apache.hadoop.ozone.web.utils;
+/**
+ * Utilities for tests.
+ */
+package org.apache.hadoop.test;
diff --git a/hadoop-hdds/tools/pom.xml b/hadoop-hdds/tools/pom.xml
index cb47eb9..f362a0b 100644
--- a/hadoop-hdds/tools/pom.xml
+++ b/hadoop-hdds/tools/pom.xml
@@ -32,6 +32,11 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-test-utils</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-common</artifactId>
</dependency>
<dependency>
@@ -44,10 +49,6 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-container-service</artifactId>
</dependency>
<dependency>
@@ -68,7 +69,6 @@
<dependency>
<groupId>org.xerial</groupId>
<artifactId>sqlite-jdbc</artifactId>
- <version>3.25.2</version>
</dependency>
</dependencies>
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
index 15f6ce8..63de95e 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
@@ -24,6 +24,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.StorageUnit;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto;
@@ -42,6 +43,7 @@
import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.tracing.TracingUtil;
+import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
import org.apache.hadoop.ipc.Client;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
@@ -51,9 +53,9 @@
import com.google.common.base.Preconditions;
import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForClients;
-import static org.apache.hadoop.hdds.utils.HddsServerUtil.getScmSecurityClient;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT;
+import static org.apache.hadoop.hdds.utils.HddsServerUtil.getScmSecurityClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -69,9 +71,14 @@
private final HddsProtos.ReplicationType replicationType;
private final StorageContainerLocationProtocol
storageContainerLocationClient;
+
+ public XceiverClientManager getXceiverClientManager() {
+ return xceiverClientManager;
+ }
+
private final XceiverClientManager xceiverClientManager;
- public ContainerOperationClient(Configuration conf) throws IOException {
+ public ContainerOperationClient(OzoneConfiguration conf) throws IOException {
storageContainerLocationClient = newContainerRpcClient(conf);
this.xceiverClientManager = newXCeiverClientManager(conf);
containerSizeB = (int) conf.getStorageSize(OZONE_SCM_CONTAINER_SIZE,
@@ -88,7 +95,7 @@
}
}
- private XceiverClientManager newXCeiverClientManager(Configuration conf)
+ private XceiverClientManager newXCeiverClientManager(ConfigurationSource conf)
throws IOException {
XceiverClientManager manager;
if (OzoneSecurityUtil.isSecurityEnabled(conf)) {
@@ -107,14 +114,15 @@
}
public static StorageContainerLocationProtocol newContainerRpcClient(
- Configuration conf) throws IOException {
+ ConfigurationSource configSource) throws IOException {
Class<StorageContainerLocationProtocolPB> protocol =
StorageContainerLocationProtocolPB.class;
-
+ Configuration conf =
+ LegacyHadoopConfigurationSource.asHadoopConfiguration(configSource);
RPC.setProtocolEngine(conf, protocol, ProtobufRpcEngine.class);
long version = RPC.getProtocolVersion(protocol);
- InetSocketAddress scmAddress = getScmAddressForClients(conf);
+ InetSocketAddress scmAddress = getScmAddressForClients(configSource);
UserGroupInformation user = UserGroupInformation.getCurrentUser();
SocketFactory socketFactory = NetUtils.getDefaultSocketFactory(conf);
int rpcTimeOut = Client.getRpcTimeout(conf);
@@ -126,7 +134,7 @@
StorageContainerLocationProtocolClientSideTranslatorPB client =
new StorageContainerLocationProtocolClientSideTranslatorPB(rpcProxy);
return TracingUtil.createProxy(
- client, StorageContainerLocationProtocol.class, conf);
+ client, StorageContainerLocationProtocol.class, configSource);
}
@Override
@@ -234,8 +242,6 @@
storageContainerLocationClient.allocateContainer(type, factor,
owner);
Pipeline pipeline = containerWithPipeline.getPipeline();
- client = xceiverClientManager.acquireClient(pipeline);
-
// connect to pipeline leader and allocate container on leader datanode.
client = xceiverClientManager.acquireClient(pipeline);
createContainer(client,
@@ -377,7 +383,7 @@
Pipeline pipeline) throws IOException {
XceiverClientSpi client = null;
try {
- client = xceiverClientManager.acquireClient(pipeline);
+ client = xceiverClientManager.acquireClientForReadData(pipeline);
ReadContainerResponseProto response =
ContainerProtocolCalls.readContainer(client, containerID, null);
if (LOG.isDebugEnabled()) {
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java
index 17da8e6..95a1235 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java
@@ -116,15 +116,27 @@
});
}
+ private String formatPortOutput(List<HddsProtos.Port> ports) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < ports.size(); i++) {
+ HddsProtos.Port port = ports.get(i);
+ sb.append(port.getName() + "=" + port.getValue());
+ if (i < ports.size() - 1) {
+ sb.append(",");
+ }
+ }
+ return sb.toString();
+ }
- // Format "ipAddress(hostName) networkLocation"
+ // Format "ipAddress(hostName):PortName1=PortValue1 networkLocation"
private void printNodesWithLocation(Collection<HddsProtos.Node> nodes) {
nodes.forEach(node -> {
System.out.print(" " + node.getNodeID().getIpAddress() + "(" +
- node.getNodeID().getHostName() + ")");
+ node.getNodeID().getHostName() + ")" +
+ ":" + formatPortOutput(node.getNodeID().getPortsList()));
System.out.println(" " +
(node.getNodeID().getNetworkLocation() != null ?
node.getNodeID().getNetworkLocation() : "NA"));
});
}
-}
\ No newline at end of file
+}
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoSubcommand.java
index 31fdb1d..ec871f9 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoSubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoSubcommand.java
@@ -22,8 +22,6 @@
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerDataProto;
import org.apache.hadoop.hdds.scm.client.ScmClient;
import org.apache.hadoop.hdds.scm.container.common.helpers
.ContainerWithPipeline;
@@ -57,26 +55,14 @@
@Override
public Void call() throws Exception {
try (ScmClient scmClient = parent.getParent().createScmClient()) {
- ContainerWithPipeline container = scmClient.
+ final ContainerWithPipeline container = scmClient.
getContainerWithPipeline(containerID);
Preconditions.checkNotNull(container, "Container cannot be null");
- ContainerDataProto containerData = scmClient.readContainer(container
- .getContainerInfo().getContainerID(), container.getPipeline());
-
// Print container report info.
LOG.info("Container id: {}", containerID);
- String openStatus =
- containerData.getState() == ContainerDataProto.State.OPEN ? "OPEN" :
- "CLOSED";
- LOG.info("Container State: {}", openStatus);
- LOG.info("Container Path: {}", containerData.getContainerPath());
-
- // Output meta data.
- String metadataStr = containerData.getMetadataList().stream().map(
- p -> p.getKey() + ":" + p.getValue())
- .collect(Collectors.joining(", "));
- LOG.info("Container Metadata: {}", metadataStr);
+ LOG.info("Pipeline id: {}", container.getPipeline().getId().getId());
+ LOG.info("Container State: {}", container.getContainerInfo().getState());
// Print pipeline of an existing container.
String machinesStr = container.getPipeline().getNodes().stream().map(
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java
index ce4610c..3ffc118 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java
@@ -25,6 +25,12 @@
import org.apache.hadoop.hdds.scm.client.ScmClient;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.PropertyAccessor;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import picocli.CommandLine.Command;
@@ -57,10 +63,23 @@
defaultValue = "20", showDefaultValue = Visibility.ALWAYS)
private int count = 20;
+ private static final ObjectWriter WRITER;
+
+ static {
+ ObjectMapper mapper = new ObjectMapper()
+ .registerModule(new JavaTimeModule())
+ .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false);
+ mapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
+ mapper
+ .setVisibility(PropertyAccessor.GETTER, JsonAutoDetect.Visibility.NONE);
+ WRITER = mapper.writerWithDefaultPrettyPrinter();
+ }
+
+
private void outputContainerInfo(ContainerInfo containerInfo)
throws IOException {
// Print container report info.
- LOG.info("{}", containerInfo.toJsonString());
+ LOG.info("{}", WRITER.writeValueAsString(containerInfo));
}
@Override
diff --git a/hadoop-ozone/Jenkinsfile b/hadoop-ozone/Jenkinsfile
deleted file mode 100644
index 0055486..0000000
--- a/hadoop-ozone/Jenkinsfile
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-node("ubuntu") {
- docker.image('elek/ozone-build').pull()
- docker.image('elek/ozone-build').inside("--privileged") {
-
- stage('Checkout') {
- checkout scm
- //use this for external Jenkinsfile builds
- //checkout poll: false, scm: [$class: 'GitSCM', branches: [[name: env.branch]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'github-token', url: "https://github.com/${organization}/${repository}.git"]]]
-
- }
-
- stage('Clean') {
- status = sh returnStatus: true, script: 'mvn clean -P hdds -am -pl :hadoop-ozone-dist '
- }
-
- stageRunner('Author', "author", {})
-
- stageRunner('Licence', "rat", {
- archiveArtifacts 'target/rat-aggregated.txt'
- }, 'artifact/target/rat-aggregated.txt/*view*/')
-
- stageRunner('Build', "build", {})
-
- stageRunner('Findbugs', "findbugs", {
- archiveArtifacts 'target/findbugs-all.txt'
-
- }, 'artifact/target/findbugs-all.txt/*view*/')
-
- stageRunner('Checkstyle', "checkstyle", {
- checkstyle canComputeNew: false, canRunOnFailed: true, defaultEncoding: '', healthy: '', pattern: '**/checkstyle-errors.xml', unHealthy: ''
- }, 'checkstyleResult')
-
- stageRunner('Acceptance', "acceptance", {
- archiveArtifacts 'hadoop-ozone/dist/target/ozone-0.4.0-SNAPSHOT/smoketest/result/**'
- })
-
- stageRunner('Unit test', "unit", {
- junit '**/target/surefire-reports/*.xml'
- }, 'testReport/')
-
- }
-
-}
-
-def stageRunner(name, type, processResult, url = '') {
- try {
- stage(name) {
- prStatusStart(type)
- status = sh returnStatus: true, script: 'hadoop-ozone/dev-support/checks/' + type + '.sh'
- processResult()
- prStatusResult(status, type, url)
- }
- return true
- } catch (RuntimeException ex) {
- currentBuild.result = "FAILED"
- return false
- }
-}
-
-def githubStatus(name, status, description, url='') {
- commitId = sh(returnStdout: true, script: 'git rev-parse HEAD')
- context = 'ci/ozone/' + name
- if (url) {
- githubNotify account: 'apache', context: context, credentialsId: 'github-pr-ozone', description: description, repo: 'hadoop', sha: commitId, status: status, targetUrl: url
- } else {
- githubNotify account: 'apache', context: context, credentialsId: 'github-pr-ozone', description: description, repo: 'hadoop', sha: commitId, status: status
- }
-}
-def prStatusStart(name) {
- githubStatus(name,
- "PENDING",
- name + " is started")
-
-
-}
-
-def prStatusResult(responseCode, name, url = '') {
- status = "ERROR"
- desc = "failed"
- if (responseCode == 0) {
- status = "SUCCESS"
- desc = "passed"
- }
- message = name + " check is " + desc
- if (url) {
- githubStatus(name,
- status,
- message,
- env.BUILD_URL + url)
- } else {
- githubStatus(name,
- status,
- message)
- }
-
- if (responseCode != 0) {
- throw new RuntimeException(message)
- }
-}
diff --git a/hadoop-ozone/client/pom.xml b/hadoop-ozone/client/pom.xml
index bac4b35..64b0aca 100644
--- a/hadoop-ozone/client/pom.xml
+++ b/hadoop-ozone/client/pom.xml
@@ -31,6 +31,11 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-test-utils</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-common</artifactId>
</dependency>
<dependency>
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
index 253aa04..b8fbe19 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -20,21 +20,19 @@
import java.io.IOException;
import java.net.URI;
-import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
-import java.util.Objects;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.hdds.tracing.TracingUtil;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.ozone.om.exceptions.OMException;
-import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
@@ -62,18 +60,24 @@
*/
private int listCacheSize;
+ private String s3VolumeName;
+
/**
* Creates an instance of ObjectStore.
* @param conf Configuration object.
* @param proxy ClientProtocol proxy.
*/
- public ObjectStore(Configuration conf, ClientProtocol proxy) {
+ public ObjectStore(ConfigurationSource conf, ClientProtocol proxy) {
this.proxy = TracingUtil.createProxy(proxy, ClientProtocol.class, conf);
this.listCacheSize = HddsClientUtils.getListCacheSize(conf);
+ this.s3VolumeName = HddsClientUtils.getS3VolumeName(conf);
}
@VisibleForTesting
protected ObjectStore() {
+ // For the unit test
+ OzoneConfiguration conf = new OzoneConfiguration();
+ this.s3VolumeName = HddsClientUtils.getS3VolumeName(conf);
proxy = null;
}
@@ -105,13 +109,17 @@
/**
* Creates an S3 bucket inside Ozone manager and creates the mapping needed
* to access via both S3 and Ozone.
- * @param userName - S3 user name.
- * @param s3BucketName - S3 bucket Name.
+ * @param bucketName - S3 bucket Name.
* @throws IOException - On failure, throws an exception like Bucket exists.
*/
- public void createS3Bucket(String userName, String s3BucketName) throws
+ public void createS3Bucket(String bucketName) throws
IOException {
- proxy.createS3Bucket(userName, s3BucketName);
+ OzoneVolume volume = getVolume(s3VolumeName);
+ volume.createBucket(bucketName);
+ }
+
+ public OzoneBucket getS3Bucket(String bucketName) throws IOException {
+ return getVolume(s3VolumeName).getBucket(bucketName);
}
/**
@@ -120,44 +128,16 @@
* @throws IOException in case the bucket cannot be deleted.
*/
public void deleteS3Bucket(String bucketName) throws IOException {
- proxy.deleteS3Bucket(bucketName);
- }
-
- /**
- * Returns the Ozone Namespace for the S3Bucket. It will return the
- * OzoneVolume/OzoneBucketName.
- * @param s3BucketName - S3 Bucket Name.
- * @return String - The Ozone canonical name for this s3 bucket. This
- * string is useful for mounting an OzoneFS.
- * @throws IOException - Error is throw if the s3bucket does not exist.
- */
- public String getOzoneBucketMapping(String s3BucketName) throws IOException {
- return proxy.getOzoneBucketMapping(s3BucketName);
- }
-
- /**
- * Returns the corresponding Ozone volume given an S3 Bucket.
- * @param s3BucketName - S3Bucket Name.
- * @return String - Ozone Volume name.
- * @throws IOException - Throws if the s3Bucket does not exist.
- */
- @SuppressWarnings("StringSplitter")
- public String getOzoneVolumeName(String s3BucketName) throws IOException {
- String mapping = getOzoneBucketMapping(s3BucketName);
- return mapping.split("/")[0];
-
- }
-
- /**
- * Returns the corresponding Ozone bucket name for the given S3 bucket.
- * @param s3BucketName - S3Bucket Name.
- * @return String - Ozone bucket Name.
- * @throws IOException - Throws if the s3bucket does not exist.
- */
- @SuppressWarnings("StringSplitter")
- public String getOzoneBucketName(String s3BucketName) throws IOException {
- String mapping = getOzoneBucketMapping(s3BucketName);
- return mapping.split("/")[1];
+ try {
+ OzoneVolume volume = getVolume(s3VolumeName);
+ volume.deleteBucket(bucketName);
+ } catch (OMException ex) {
+ if (ex.getResult() == OMException.ResultCodes.VOLUME_NOT_FOUND) {
+ throw new OMException(OMException.ResultCodes.BUCKET_NOT_FOUND);
+ } else {
+ throw ex;
+ }
+ }
}
@@ -177,37 +157,6 @@
}
/**
- * Returns Iterator to iterate over all buckets for a user.
- * The result can be restricted using bucket prefix, will return all
- * buckets if bucket prefix is null.
- *
- * @param userName user name
- * @param bucketPrefix Bucket prefix to match
- * @return {@code Iterator<OzoneBucket>}
- */
- public Iterator<? extends OzoneBucket> listS3Buckets(String userName,
- String bucketPrefix) {
- return listS3Buckets(userName, bucketPrefix, null);
- }
-
- /**
- * Returns Iterator to iterate over all buckets after prevBucket for a
- * specific user. If prevBucket is null it returns an iterator to iterate over
- * all the buckets of a user. The result can be restricted using bucket
- * prefix, will return all buckets if bucket prefix is null.
- *
- * @param userName user name
- * @param bucketPrefix Bucket prefix to match
- * @param prevBucket Buckets are listed after this bucket
- * @return {@code Iterator<OzoneBucket>}
- */
- public Iterator<? extends OzoneBucket> listS3Buckets(String userName,
- String bucketPrefix,
- String prevBucket) {
- return new S3BucketIterator(userName, bucketPrefix, prevBucket);
- }
-
- /**
* Returns Iterator to iterate over all the volumes in object store.
* The result can be restricted using volume prefix, will return all
* volumes if volume prefix is null.
@@ -250,7 +199,7 @@
String volumePrefix, String prevVolume)
throws IOException {
if(Strings.isNullOrEmpty(user)) {
- user = UserGroupInformation.getCurrentUser().getShortUserName();
+ user = UserGroupInformation.getCurrentUser().getUserName();
}
return new VolumeIterator(user, volumePrefix, prevVolume);
}
@@ -334,76 +283,6 @@
}
/**
- * An Iterator to iterate over {@link OzoneBucket} list.
- */
- public class S3BucketIterator implements Iterator<OzoneBucket> {
-
- private String bucketPrefix = null;
- private String userName;
-
- private Iterator<OzoneBucket> currentIterator;
- private OzoneBucket currentValue;
-
-
- /**
- * Creates an Iterator to iterate over all buckets after prevBucket for
- * a user. If prevBucket is null it returns an iterator which list all
- * the buckets of the user.
- * The returned buckets match bucket prefix.
- * @param user
- * @param bucketPrefix
- * @param prevBucket
- */
- public S3BucketIterator(String user, String bucketPrefix, String
- prevBucket) {
- Objects.requireNonNull(user);
- this.userName = user;
- this.bucketPrefix = bucketPrefix;
- this.currentValue = null;
- this.currentIterator = getNextListOfS3Buckets(prevBucket).iterator();
- }
-
- @Override
- public boolean hasNext() {
- if(!currentIterator.hasNext()) {
- currentIterator = getNextListOfS3Buckets(
- currentValue != null ? currentValue.getName() : null)
- .iterator();
- }
- return currentIterator.hasNext();
- }
-
- @Override
- public OzoneBucket next() {
- if(hasNext()) {
- currentValue = currentIterator.next();
- return currentValue;
- }
- throw new NoSuchElementException();
- }
-
- /**
- * Gets the next set of bucket list using proxy.
- * @param prevBucket
- * @return {@code List<OzoneVolume>}
- */
- private List<OzoneBucket> getNextListOfS3Buckets(String prevBucket) {
- try {
- return proxy.listS3Buckets(userName, bucketPrefix, prevBucket,
- listCacheSize);
- } catch (OMException e) {
- if (e.getResult() == ResultCodes.VOLUME_NOT_FOUND) {
- return new ArrayList<>();
- } else {
- throw new RuntimeException(e);
- }
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- /**
* Get a valid Delegation Token.
*
* @param renewer the designated renewer for the token
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
index e766ebd..87710ea 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
@@ -21,7 +21,7 @@
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
@@ -104,7 +104,7 @@
private OzoneObj ozoneObj;
- private OzoneBucket(Configuration conf, String volumeName,
+ private OzoneBucket(ConfigurationSource conf, String volumeName,
String bucketName, ReplicationFactor defaultReplication,
ReplicationType defaultReplicationType, ClientProtocol proxy) {
Preconditions.checkNotNull(proxy, "Client proxy is not set.");
@@ -133,7 +133,7 @@
.setStoreType(OzoneObj.StoreType.OZONE).build();
}
@SuppressWarnings("parameternumber")
- public OzoneBucket(Configuration conf, ClientProtocol proxy,
+ public OzoneBucket(ConfigurationSource conf, ClientProtocol proxy,
String volumeName, String bucketName, StorageType storageType,
Boolean versioning, long creationTime, Map<String, String> metadata,
String encryptionKeyName) {
@@ -157,7 +157,7 @@
* @param creationTime creation time of the bucket.
*/
@SuppressWarnings("parameternumber")
- public OzoneBucket(Configuration conf, ClientProtocol proxy,
+ public OzoneBucket(ConfigurationSource conf, ClientProtocol proxy,
String volumeName, String bucketName, StorageType storageType,
Boolean versioning, long creationTime, Map<String, String> metadata) {
this(conf, volumeName, bucketName, null, null, proxy);
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClient.java
index 0d65d73..493315e 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClient.java
@@ -18,7 +18,8 @@
package org.apache.hadoop.ozone.client;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import java.io.Closeable;
@@ -74,6 +75,7 @@
private final ClientProtocol proxy;
private final ObjectStore objectStore;
+ private ConfigurationSource conf;
/**
* Creates a new OzoneClient object, generally constructed
@@ -81,15 +83,18 @@
* @param conf Configuration object
* @param proxy ClientProtocol proxy instance
*/
- public OzoneClient(Configuration conf, ClientProtocol proxy) {
+ public OzoneClient(ConfigurationSource conf, ClientProtocol proxy) {
this.proxy = proxy;
this.objectStore = new ObjectStore(conf, this.proxy);
+ this.conf = conf;
}
@VisibleForTesting
protected OzoneClient(ObjectStore objectStore) {
this.objectStore = objectStore;
this.proxy = null;
+ // For the unit test
+ this.conf = new OzoneConfiguration();
}
/**
* Returns the object store associated with the Ozone Cluster.
@@ -100,6 +105,14 @@
}
/**
+ * Returns the configuration of client.
+ * @return ConfigurationSource
+ */
+ public ConfigurationSource getConfiguration() {
+ return conf;
+ }
+
+ /**
* Closes the client and all the underlying resources.
* @throws IOException
*/
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java
index 8417347..8e4882a 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java
@@ -18,19 +18,25 @@
package org.apache.hadoop.ozone.client;
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
import java.io.IOException;
import java.lang.reflect.Proxy;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.ozone.client.rpc.RpcClient;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
import com.google.common.base.Preconditions;
+import org.apache.commons.lang3.StringUtils;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -77,7 +83,7 @@
* @throws IOException
*/
public static OzoneClient getRpcClient(String omHost, Integer omRpcPort,
- Configuration config)
+ ConfigurationSource config)
throws IOException {
Preconditions.checkNotNull(omHost);
Preconditions.checkNotNull(omRpcPort);
@@ -100,7 +106,7 @@
* @throws IOException
*/
public static OzoneClient getRpcClient(String omServiceId,
- Configuration config) throws IOException {
+ ConfigurationSource config) throws IOException {
Preconditions.checkNotNull(omServiceId);
Preconditions.checkNotNull(config);
if (OmUtils.isOmHAServiceId(config, omServiceId)) {
@@ -123,7 +129,7 @@
*
* @throws IOException
*/
- public static OzoneClient getRpcClient(Configuration config)
+ public static OzoneClient getRpcClient(ConfigurationSource config)
throws IOException {
Preconditions.checkNotNull(config);
@@ -150,7 +156,7 @@
* Configuration to be used for OzoneClient creation
*/
private static OzoneClient getRpcClient(ClientProtocol clientProtocol,
- Configuration config) {
+ ConfigurationSource config) {
OzoneClientInvocationHandler clientHandler =
new OzoneClientInvocationHandler(clientProtocol);
ClientProtocol proxy = (ClientProtocol) Proxy.newProxyInstance(
@@ -160,6 +166,55 @@
}
/**
+ * Create OzoneClient for token renew/cancel operations.
+ * @param conf Configuration to be used for OzoneCient creation
+ * @param token ozone token is involved
+ * @return
+ * @throws IOException
+ */
+ public static OzoneClient getOzoneClient(Configuration conf,
+ Token<OzoneTokenIdentifier> token) throws IOException {
+ Preconditions.checkNotNull(token, "Null token is not allowed");
+ OzoneTokenIdentifier tokenId = new OzoneTokenIdentifier();
+ ByteArrayInputStream buf = new ByteArrayInputStream(
+ token.getIdentifier());
+ DataInputStream in = new DataInputStream(buf);
+ tokenId.readFields(in);
+ String omServiceId = tokenId.getOmServiceId();
+ OzoneConfiguration ozoneConf = OzoneConfiguration.of(conf);
+ // Must check with OzoneConfiguration so that ozone-site.xml is loaded.
+ if (StringUtils.isNotEmpty(omServiceId)) {
+ // new OM should always issue token with omServiceId
+ if (!OmUtils.isServiceIdsDefined(ozoneConf)
+ && omServiceId.equals(OzoneConsts.OM_SERVICE_ID_DEFAULT)) {
+ // Non-HA or single-node Ratis HA
+ return OzoneClientFactory.getRpcClient(ozoneConf);
+ } else if (OmUtils.isOmHAServiceId(ozoneConf, omServiceId)) {
+ // HA with matching service id
+ return OzoneClientFactory.getRpcClient(omServiceId, ozoneConf);
+ } else {
+ // HA with mismatched service id
+ throw new IOException("Service ID specified " + omServiceId +
+ " does not match" + " with " + OZONE_OM_SERVICE_IDS_KEY +
+ " defined in the " + "configuration. Configured " +
+ OZONE_OM_SERVICE_IDS_KEY + " are" +
+ ozoneConf.getTrimmedStringCollection(OZONE_OM_SERVICE_IDS_KEY));
+ }
+ } else {
+ // Old OM may issue token without omServiceId that should work
+ // with non-HA case
+ if (!OmUtils.isServiceIdsDefined(ozoneConf)) {
+ return OzoneClientFactory.getRpcClient(ozoneConf);
+ } else {
+ throw new IOException("OzoneToken with no service ID can't "
+ + "be renewed or canceled with local OM HA setup because we "
+ + "don't know if the token is issued from local OM HA cluster "
+ + "or not.");
+ }
+ }
+ }
+
+ /**
* Returns an instance of Protocol class.
*
*
@@ -170,7 +225,7 @@
*
* @throws IOException
*/
- private static ClientProtocol getClientProtocol(Configuration config)
+ private static ClientProtocol getClientProtocol(ConfigurationSource config)
throws IOException {
return getClientProtocol(config, null);
}
@@ -186,7 +241,7 @@
*
* @throws IOException
*/
- private static ClientProtocol getClientProtocol(Configuration config,
+ private static ClientProtocol getClientProtocol(ConfigurationSource config,
String omServiceId) throws IOException {
try {
return new RpcClient(config, omServiceId);
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java
index cca923a..07c1e26 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java
@@ -26,7 +26,7 @@
import java.util.Map;
import java.util.NoSuchElementException;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.client.OzoneQuota;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.ozone.OzoneAcl;
@@ -88,10 +88,11 @@
* @param metadata custom key value metadata.
*/
@SuppressWarnings("parameternumber")
- public OzoneVolume(Configuration conf, ClientProtocol proxy, String name,
- String admin, String owner, long quotaInBytes,
- long creationTime, List<OzoneAcl> acls,
- Map<String, String> metadata) {
+ public OzoneVolume(ConfigurationSource conf, ClientProtocol proxy,
+ String name,
+ String admin, String owner, long quotaInBytes,
+ long creationTime, List<OzoneAcl> acls,
+ Map<String, String> metadata) {
Preconditions.checkNotNull(proxy, "Client proxy is not set.");
this.proxy = proxy;
this.name = name;
@@ -105,9 +106,10 @@
}
@SuppressWarnings("parameternumber")
- public OzoneVolume(Configuration conf, ClientProtocol proxy, String name,
- String admin, String owner, long quotaInBytes,
- long creationTime, List<OzoneAcl> acls) {
+ public OzoneVolume(ConfigurationSource conf, ClientProtocol proxy,
+ String name,
+ String admin, String owner, long quotaInBytes,
+ long creationTime, List<OzoneAcl> acls) {
this(conf, proxy, name, admin, owner, quotaInBytes, creationTime, acls,
new HashMap<>());
}
@@ -182,12 +184,13 @@
/**
* Sets/Changes the owner of this Volume.
- * @param owner new owner
+ * @param userName new owner
* @throws IOException
*/
- public void setOwner(String owner) throws IOException {
- proxy.setVolumeOwner(name, owner);
- this.owner = owner;
+ public boolean setOwner(String userName) throws IOException {
+ boolean result = proxy.setVolumeOwner(name, userName);
+ this.owner = userName;
+ return result;
}
/**
@@ -195,7 +198,7 @@
* @param quota new quota
* @throws IOException
*/
- public void setQuota(OzoneQuota quota) throws IOException {
+ public void setQuota(OzoneQuota quota) throws IOException {
proxy.setVolumeQuota(name, quota);
this.quotaInBytes = quota.sizeInBytes();
}
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockOutputStreamEntry.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockOutputStreamEntry.java
index 40a4a6b..8e1e640 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockOutputStreamEntry.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockOutputStreamEntry.java
@@ -55,7 +55,9 @@
private long currentPosition;
private Token<OzoneBlockTokenIdentifier> token;
+ private final int streamBufferSize;
private final long streamBufferFlushSize;
+ private final boolean streamBufferFlushDelay;
private final long streamBufferMaxSize;
private final long watchTimeout;
private BufferPool bufferPool;
@@ -64,7 +66,8 @@
private BlockOutputStreamEntry(BlockID blockID, String key,
XceiverClientManager xceiverClientManager,
Pipeline pipeline, String requestId, int chunkSize,
- long length, long streamBufferFlushSize, long streamBufferMaxSize,
+ long length, int streamBufferSize, long streamBufferFlushSize,
+ boolean streamBufferFlushDelay, long streamBufferMaxSize,
long watchTimeout, BufferPool bufferPool,
ChecksumType checksumType, int bytesPerChecksum,
Token<OzoneBlockTokenIdentifier> token) {
@@ -77,7 +80,9 @@
this.token = token;
this.length = length;
this.currentPosition = 0;
+ this.streamBufferSize = streamBufferSize;
this.streamBufferFlushSize = streamBufferFlushSize;
+ this.streamBufferFlushDelay = streamBufferFlushDelay;
this.streamBufferMaxSize = streamBufferMaxSize;
this.watchTimeout = watchTimeout;
this.bufferPool = bufferPool;
@@ -110,9 +115,9 @@
}
this.outputStream =
new BlockOutputStream(blockID, xceiverClientManager,
- pipeline, streamBufferFlushSize,
- streamBufferMaxSize, bufferPool, checksumType,
- bytesPerChecksum);
+ pipeline, streamBufferSize, streamBufferFlushSize,
+ streamBufferFlushDelay, streamBufferMaxSize, bufferPool,
+ checksumType, bytesPerChecksum);
}
}
@@ -215,7 +220,9 @@
private String requestId;
private int chunkSize;
private long length;
+ private int streamBufferSize;
private long streamBufferFlushSize;
+ private boolean streamBufferFlushDelay;
private long streamBufferMaxSize;
private long watchTimeout;
private BufferPool bufferPool;
@@ -269,11 +276,21 @@
return this;
}
+ public Builder setStreamBufferSize(int bufferSize) {
+ this.streamBufferSize = bufferSize;
+ return this;
+ }
+
public Builder setStreamBufferFlushSize(long bufferFlushSize) {
this.streamBufferFlushSize = bufferFlushSize;
return this;
}
+ public Builder setStreamBufferFlushDelay(boolean bufferFlushDelay) {
+ this.streamBufferFlushDelay = bufferFlushDelay;
+ return this;
+ }
+
public Builder setStreamBufferMaxSize(long bufferMaxSize) {
this.streamBufferMaxSize = bufferMaxSize;
return this;
@@ -297,7 +314,8 @@
public BlockOutputStreamEntry build() {
return new BlockOutputStreamEntry(blockID, key,
xceiverClientManager, pipeline, requestId, chunkSize,
- length, streamBufferFlushSize, streamBufferMaxSize, watchTimeout,
+ length, streamBufferSize, streamBufferFlushSize,
+ streamBufferFlushDelay, streamBufferMaxSize, watchTimeout,
bufferPool, checksumType, bytesPerChecksum, token);
}
}
@@ -331,10 +349,18 @@
return currentPosition;
}
+ public int getStreamBufferSize() {
+ return streamBufferSize;
+ }
+
public long getStreamBufferFlushSize() {
return streamBufferFlushSize;
}
+ public boolean getStreamBufferFlushDelay() {
+ return streamBufferFlushDelay;
+ }
+
public long getStreamBufferMaxSize() {
return streamBufferMaxSize;
}
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockOutputStreamEntryPool.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockOutputStreamEntryPool.java
index 17683ad..3cab664 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockOutputStreamEntryPool.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockOutputStreamEntryPool.java
@@ -61,6 +61,7 @@
private final String requestID;
private final int streamBufferSize;
private final long streamBufferFlushSize;
+ private final boolean streamBufferFlushDelay;
private final long streamBufferMaxSize;
private final long watchTimeout;
private final long blockSize;
@@ -75,7 +76,8 @@
public BlockOutputStreamEntryPool(OzoneManagerProtocol omClient,
int chunkSize, String requestId, HddsProtos.ReplicationFactor factor,
HddsProtos.ReplicationType type,
- int bufferSize, long bufferFlushSize, long bufferMaxSize,
+ int bufferSize, long bufferFlushSize,
+ boolean bufferFlushDelay, long bufferMaxSize,
long size, long watchTimeout, ContainerProtos.ChecksumType checksumType,
int bytesPerChecksum, String uploadID, int partNumber,
boolean isMultipart, OmKeyInfo info,
@@ -93,6 +95,7 @@
this.requestID = requestId;
this.streamBufferSize = bufferSize;
this.streamBufferFlushSize = bufferFlushSize;
+ this.streamBufferFlushDelay = bufferFlushDelay;
this.streamBufferMaxSize = bufferMaxSize;
this.blockSize = size;
this.watchTimeout = watchTimeout;
@@ -137,6 +140,7 @@
requestID = null;
streamBufferSize = 0;
streamBufferFlushSize = 0;
+ streamBufferFlushDelay = false;
streamBufferMaxSize = 0;
bufferPool = new BufferPool(chunkSize, 1);
watchTimeout = 0;
@@ -188,7 +192,9 @@
.setRequestId(requestID)
.setChunkSize(chunkSize)
.setLength(subKeyInfo.getLength())
+ .setStreamBufferSize(streamBufferSize)
.setStreamBufferFlushSize(streamBufferFlushSize)
+ .setStreamBufferFlushDelay(streamBufferFlushDelay)
.setStreamBufferMaxSize(streamBufferMaxSize)
.setWatchTimeout(watchTimeout)
.setbufferPool(bufferPool)
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyInputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyInputStream.java
index 8e375bf..4af6838 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyInputStream.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyInputStream.java
@@ -20,6 +20,7 @@
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FSExceptionMessages;
import org.apache.hadoop.fs.Seekable;
import org.apache.hadoop.hdds.client.BlockID;
@@ -34,6 +35,7 @@
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -322,4 +324,63 @@
public long getRemainingOfIndex(int index) throws IOException {
return blockStreams.get(index).getRemaining();
}
+
+ /**
+ * Copies some or all bytes from a large (over 2GB) <code>InputStream</code>
+ * to an <code>OutputStream</code>, optionally skipping input bytes.
+ * <p>
+ * Copy the method from IOUtils of commons-io to reimplement skip by seek
+ * rather than read. The reason why IOUtils of commons-io implement skip
+ * by read can be found at
+ * <a href="https://issues.apache.org/jira/browse/IO-203">IO-203</a>.
+ * </p>
+ * <p>
+ * This method uses the provided buffer, so there is no need to use a
+ * <code>BufferedInputStream</code>.
+ * </p>
+ *
+ * @param output the <code>OutputStream</code> to write to
+ * @param inputOffset : number of bytes to skip from input before copying
+ * -ve values are ignored
+ * @param length : number of bytes to copy. -ve means all
+ * @param buffer the buffer to use for the copy
+ * @return the number of bytes copied
+ * @throws NullPointerException if the input or output is null
+ * @throws IOException if an I/O error occurs
+ */
+ public long copyLarge(final OutputStream output,
+ final long inputOffset, final long len, final byte[] buffer)
+ throws IOException {
+ if (inputOffset > 0) {
+ seek(inputOffset);
+ }
+
+ if (len == 0) {
+ return 0;
+ }
+
+ final int bufferLength = buffer.length;
+ int bytesToRead = bufferLength;
+ if (len > 0 && len < bufferLength) {
+ bytesToRead = (int) len;
+ }
+
+ int read;
+ long totalRead = 0;
+ while (bytesToRead > 0) {
+ read = read(buffer, 0, bytesToRead);
+ if (read == IOUtils.EOF) {
+ break;
+ }
+
+ output.write(buffer, 0, read);
+ totalRead += read;
+ if (len > 0) { // only adjust len if not reading to the end
+ // Note the cast must work because buffer.length is an integer
+ bytesToRead = (int) Math.min(len - totalRead, bufferLength);
+ }
+ }
+
+ return totalRead;
+ }
}
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStream.java
index ba0ba121..843155c 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStream.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStream.java
@@ -124,15 +124,16 @@
XceiverClientManager xceiverClientManager,
OzoneManagerProtocol omClient, int chunkSize,
String requestId, ReplicationFactor factor, ReplicationType type,
- int bufferSize, long bufferFlushSize, long bufferMaxSize,
- long size, long watchTimeout,
+ int bufferSize, long bufferFlushSize, boolean isBufferFlushDelay,
+ long bufferMaxSize, long size, long watchTimeout,
ChecksumType checksumType, int bytesPerChecksum,
String uploadID, int partNumber, boolean isMultipart,
int maxRetryCount, long retryInterval) {
OmKeyInfo info = handler.getKeyInfo();
blockOutputStreamEntryPool =
new BlockOutputStreamEntryPool(omClient, chunkSize, requestId, factor,
- type, bufferSize, bufferFlushSize, bufferMaxSize, size,
+ type, bufferSize, bufferFlushSize, isBufferFlushDelay,
+ bufferMaxSize, size,
watchTimeout, checksumType, bytesPerChecksum, uploadID, partNumber,
isMultipart, info, xceiverClientManager, handler.getId());
// Retrieve the file encryption key info, null if file is not in
@@ -542,6 +543,7 @@
private ReplicationFactor factor;
private int streamBufferSize;
private long streamBufferFlushSize;
+ private boolean streamBufferFlushDelay;
private long streamBufferMaxSize;
private long blockSize;
private long watchTimeout;
@@ -608,6 +610,11 @@
return this;
}
+ public Builder setStreamBufferFlushDelay(boolean isDelay) {
+ this.streamBufferFlushDelay = isDelay;
+ return this;
+ }
+
public Builder setStreamBufferMaxSize(long size) {
this.streamBufferMaxSize = size;
return this;
@@ -646,7 +653,8 @@
public KeyOutputStream build() {
return new KeyOutputStream(openHandler, xceiverManager, omClient,
chunkSize, requestID, factor, type,
- streamBufferSize, streamBufferFlushSize, streamBufferMaxSize,
+ streamBufferSize, streamBufferFlushSize, streamBufferFlushDelay,
+ streamBufferMaxSize,
blockSize, watchTimeout, checksumType,
bytesPerChecksum, multipartUploadID, multipartNumber, isMultipartKey,
maxRetryCount, retryInterval);
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index bdeaf95..06288a0 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -88,9 +88,11 @@
* Sets the owner of volume.
* @param volumeName Name of the Volume
* @param owner to be set for the Volume
+ * @return true if operation succeeded, false if specified user is
+ * already the owner.
* @throws IOException
*/
- void setVolumeOwner(String volumeName, String owner) throws IOException;
+ boolean setVolumeOwner(String volumeName, String owner) throws IOException;
/**
* Set Volume Quota.
@@ -342,7 +344,7 @@
* @param bucketName - The bucket name.
* @param keyName - The key user want to recover.
* @param destinationBucket - The bucket user want to recover to.
- * @return The recoverTrash
+ * @return The result of recovering operation is success or not.
* @throws IOException
*/
boolean recoverTrash(String volumeName, String bucketName, String keyName,
@@ -361,65 +363,6 @@
throws IOException;
/**
- * Creates an S3 bucket inside Ozone manager and creates the mapping needed
- * to access via both S3 and Ozone.
- * @param userName - S3 user name.
- * @param s3BucketName - S3 bucket Name.
- * @throws IOException - On failure, throws an exception like Bucket exists.
- */
- void createS3Bucket(String userName, String s3BucketName) throws IOException;
-
- /**
- * Deletes an s3 bucket and removes mapping of Ozone volume/bucket.
- * @param bucketName - S3 Bucket Name.
- * @throws IOException in case the bucket cannot be deleted.
- */
- void deleteS3Bucket(String bucketName) throws IOException;
-
-
- /**
- * Returns the Ozone Namespace for the S3Bucket. It will return the
- * OzoneVolume/OzoneBucketName.
- * @param s3BucketName - S3 Bucket Name.
- * @return String - The Ozone canonical name for this s3 bucket. This
- * string is useful for mounting an OzoneFS.
- * @throws IOException - Error is throw if the s3bucket does not exist.
- */
- String getOzoneBucketMapping(String s3BucketName) throws IOException;
-
- /**
- * Returns the corresponding Ozone volume given an S3 Bucket.
- * @param s3BucketName - S3Bucket Name.
- * @return String - Ozone Volume name.
- * @throws IOException - Throws if the s3Bucket does not exist.
- */
- String getOzoneVolumeName(String s3BucketName) throws IOException;
-
- /**
- * Returns the corresponding Ozone bucket name for the given S3 bucket.
- * @param s3BucketName - S3Bucket Name.
- * @return String - Ozone bucket Name.
- * @throws IOException - Throws if the s3bucket does not exist.
- */
- String getOzoneBucketName(String s3BucketName) throws IOException;
-
- /**
- * Returns Iterator to iterate over all buckets after prevBucket for a
- * specific user. If prevBucket is null it returns an iterator to iterate over
- * all the buckets of a user. The result can be restricted using bucket
- * prefix, will return all buckets if bucket prefix is null.
- *
- * @param userName user name
- * @param bucketPrefix Bucket prefix to match
- * @param prevBucket Buckets are listed after this bucket
- * @return {@code Iterator<OzoneBucket>}
- * @throws IOException
- */
- List<OzoneBucket> listS3Buckets(String userName, String bucketPrefix,
- String prevBucket, int maxListResult)
- throws IOException;
-
- /**
* Close and release the resources.
*/
void close() throws IOException;
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/OzoneKMSUtil.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/OzoneKMSUtil.java
index 6be7770..84c341d 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/OzoneKMSUtil.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/OzoneKMSUtil.java
@@ -18,6 +18,12 @@
package org.apache.hadoop.ozone.client.rpc;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.CipherSuite;
import org.apache.hadoop.crypto.CryptoCodec;
@@ -26,6 +32,8 @@
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
import org.apache.hadoop.fs.FileEncryptionInfo;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ozone.om.exceptions.OMException;
@@ -33,12 +41,6 @@
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.KMSUtil;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URI;
-import java.nio.charset.StandardCharsets;
-import java.security.GeneralSecurityException;
-
/**
* KMS utility class for Ozone Data Encryption At-Rest.
*/
@@ -94,7 +96,7 @@
}
public static URI getKeyProviderUri(UserGroupInformation ugi,
- URI namespaceUri, String kmsUriSrv, Configuration conf)
+ URI namespaceUri, String kmsUriSrv, ConfigurationSource conf)
throws IOException {
URI keyProviderUri = null;
Credentials credentials = ugi.getCredentials();
@@ -110,8 +112,10 @@
if (keyProviderUri == null) {
// from client conf
if (kmsUriSrv == null) {
+ Configuration hadoopConfig =
+ LegacyHadoopConfigurationSource.asHadoopConfiguration(conf);
keyProviderUri = KMSUtil.getKeyProviderUri(
- conf, keyProviderUriKeyName);
+ hadoopConfig, keyProviderUriKeyName);
} else if (!kmsUriSrv.isEmpty()) {
// from om server
keyProviderUri = URI.create(kmsUriSrv);
@@ -126,12 +130,14 @@
return keyProviderUri;
}
- public static KeyProvider getKeyProvider(final Configuration conf,
+ public static KeyProvider getKeyProvider(final ConfigurationSource conf,
final URI serverProviderUri) throws IOException{
if (serverProviderUri == null) {
throw new IOException("KMS serverProviderUri is not configured.");
}
- return KMSUtil.createKeyProviderFromUri(conf, serverProviderUri);
+ return KMSUtil.createKeyProviderFromUri(
+ LegacyHadoopConfigurationSource.asHadoopConfiguration(conf),
+ serverProviderUri);
}
public static CryptoProtocolVersion getCryptoProtocolVersion(
@@ -156,14 +162,16 @@
}
}
- public static CryptoCodec getCryptoCodec(Configuration conf,
+ public static CryptoCodec getCryptoCodec(ConfigurationSource conf,
FileEncryptionInfo feInfo) throws IOException {
CipherSuite suite = feInfo.getCipherSuite();
if (suite.equals(CipherSuite.UNKNOWN)) {
throw new IOException("NameNode specified unknown CipherSuite with ID " +
suite.getUnknownValue() + ", cannot instantiate CryptoCodec.");
} else {
- CryptoCodec codec = CryptoCodec.getInstance(conf, suite);
+ Configuration hadoopConfig =
+ LegacyHadoopConfigurationSource.asHadoopConfiguration(conf);
+ CryptoCodec codec = CryptoCodec.getInstance(hadoopConfig, suite);
if (codec == null) {
throw new OMException("No configuration found for the cipher suite " +
suite.getConfigSuffix() + " prefixed with " +
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index ab930e8..10d1bad 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -18,28 +18,53 @@
package org.apache.hadoop.ozone.client.rpc;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.StorageUnit;
+import javax.crypto.Cipher;
+import javax.crypto.CipherInputStream;
+import javax.crypto.CipherOutputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.security.InvalidKeyException;
+import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
import org.apache.hadoop.crypto.CryptoInputStream;
import org.apache.hadoop.crypto.CryptoOutputStream;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.fs.FileEncryptionInfo;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.StorageType;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ChecksumType;
-import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
-import org.apache.hadoop.hdds.tracing.TracingUtil;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.OzoneSecurityUtil;
-import org.apache.hadoop.ozone.client.*;
import org.apache.hadoop.hdds.client.OzoneQuota;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.StorageType;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.XceiverClientManager;
+import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
+import org.apache.hadoop.hdds.tracing.TracingUtil;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OzoneAcl;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.OzoneSecurityUtil;
+import org.apache.hadoop.ozone.client.BucketArgs;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.client.OzoneKeyLocation;
+import org.apache.hadoop.ozone.client.OzoneMultipartUpload;
+import org.apache.hadoop.ozone.client.OzoneMultipartUploadList;
+import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts;
+import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.VolumeArgs;
import org.apache.hadoop.ozone.client.io.KeyInputStream;
import org.apache.hadoop.ozone.client.io.KeyOutputStream;
@@ -69,12 +94,7 @@
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
-import org.apache.hadoop.ozone.om.protocolPB
- .OzoneManagerProtocolClientSideTranslatorPB;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.XceiverClientManager;
+import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRoleInfo;
import org.apache.hadoop.ozone.security.GDPRSymmetricKey;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
@@ -85,25 +105,15 @@
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.io.Text;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS;
import org.apache.logging.log4j.util.Strings;
import org.apache.ratis.protocol.ClientId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.crypto.Cipher;
-import javax.crypto.CipherInputStream;
-import javax.crypto.CipherOutputStream;
-import java.io.IOException;
-import java.net.URI;
-import java.security.InvalidKeyException;
-import java.security.SecureRandom;
-import java.util.*;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-
-import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS;
-
/**
* Ozone RPC Client Implementation, it connects to OM, SCM and DataNode
* to execute client calls. This uses RPC protocol for communication
@@ -114,7 +124,7 @@
private static final Logger LOG =
LoggerFactory.getLogger(RpcClient.class);
- private final OzoneConfiguration conf;
+ private final ConfigurationSource conf;
private final OzoneManagerProtocol ozoneManagerClient;
private final XceiverClientManager xceiverClientManager;
private final int chunkSize;
@@ -126,6 +136,7 @@
private final ACLType groupRights;
private final int streamBufferSize;
private final long streamBufferFlushSize;
+ private boolean streamBufferFlushDelay;
private final long streamBufferMaxSize;
private final long blockSize;
private final ClientId clientId = ClientId.randomId();
@@ -140,9 +151,10 @@
* @param omServiceId OM HA Service ID, set this to null if not HA
* @throws IOException
*/
- public RpcClient(Configuration conf, String omServiceId) throws IOException {
+ public RpcClient(ConfigurationSource conf, String omServiceId)
+ throws IOException {
Preconditions.checkNotNull(conf);
- this.conf = new OzoneConfiguration(conf);
+ this.conf = conf;
this.ugi = UserGroupInformation.getCurrentUser();
// Get default acl rights for user and group.
OzoneAclConfig aclConfig = this.conf.getObject(OzoneAclConfig.class);
@@ -185,6 +197,9 @@
.getStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT,
StorageUnit.BYTES);
+ streamBufferFlushDelay = conf.getBoolean(
+ OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_DELAY,
+ OzoneConfigKeys.OOZONE_CLIENT_STREAM_BUFFER_FLUSH_DELAY_DEFAULT);
streamBufferMaxSize = (long) conf
.getStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT,
@@ -252,7 +267,7 @@
@Override
public void createVolume(String volumeName, VolumeArgs volArgs)
throws IOException {
- HddsClientUtils.verifyResourceName(volumeName);
+ verifyVolumeName(volumeName);
Preconditions.checkNotNull(volArgs);
String admin = volArgs.getAdmin() == null ?
@@ -299,17 +314,17 @@
}
@Override
- public void setVolumeOwner(String volumeName, String owner)
+ public boolean setVolumeOwner(String volumeName, String owner)
throws IOException {
- HddsClientUtils.verifyResourceName(volumeName);
+ verifyVolumeName(volumeName);
Preconditions.checkNotNull(owner);
- ozoneManagerClient.setOwner(volumeName, owner);
+ return ozoneManagerClient.setOwner(volumeName, owner);
}
@Override
public void setVolumeQuota(String volumeName, OzoneQuota quota)
throws IOException {
- HddsClientUtils.verifyResourceName(volumeName);
+ verifyVolumeName(volumeName);
Preconditions.checkNotNull(quota);
long quotaInBytes = quota.sizeInBytes();
ozoneManagerClient.setQuota(volumeName, quotaInBytes);
@@ -318,7 +333,7 @@
@Override
public OzoneVolume getVolumeDetails(String volumeName)
throws IOException {
- HddsClientUtils.verifyResourceName(volumeName);
+ verifyVolumeName(volumeName);
OmVolumeArgs volume = ozoneManagerClient.getVolumeInfo(volumeName);
return new OzoneVolume(
conf,
@@ -341,7 +356,7 @@
@Override
public void deleteVolume(String volumeName) throws IOException {
- HddsClientUtils.verifyResourceName(volumeName);
+ verifyVolumeName(volumeName);
ozoneManagerClient.deleteVolume(volumeName);
}
@@ -398,7 +413,8 @@
public void createBucket(
String volumeName, String bucketName, BucketArgs bucketArgs)
throws IOException {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
+ verifyVolumeName(volumeName);
+ verifyBucketName(bucketName);
Preconditions.checkNotNull(bucketArgs);
Boolean isVersionEnabled = bucketArgs.getVersioning() == null ?
@@ -435,6 +451,24 @@
ozoneManagerClient.createBucket(builder.build());
}
+ private static void verifyVolumeName(String volumeName) throws OMException {
+ try {
+ HddsClientUtils.verifyResourceName(volumeName);
+ } catch (IllegalArgumentException e) {
+ throw new OMException(e.getMessage(),
+ OMException.ResultCodes.INVALID_VOLUME_NAME);
+ }
+ }
+
+ private static void verifyBucketName(String bucketName) throws OMException {
+ try {
+ HddsClientUtils.verifyResourceName(bucketName);
+ } catch (IllegalArgumentException e) {
+ throw new OMException(e.getMessage(),
+ OMException.ResultCodes.INVALID_BUCKET_NAME);
+ }
+ }
+
/**
* Helper function to get default acl list for current user.
*
@@ -521,7 +555,8 @@
public void setBucketVersioning(
String volumeName, String bucketName, Boolean versioning)
throws IOException {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
+ verifyVolumeName(volumeName);
+ verifyBucketName(bucketName);
Preconditions.checkNotNull(versioning);
OmBucketArgs.Builder builder = OmBucketArgs.newBuilder();
builder.setVolumeName(volumeName)
@@ -534,7 +569,8 @@
public void setBucketStorageType(
String volumeName, String bucketName, StorageType storageType)
throws IOException {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
+ verifyVolumeName(volumeName);
+ verifyBucketName(bucketName);
Preconditions.checkNotNull(storageType);
OmBucketArgs.Builder builder = OmBucketArgs.newBuilder();
builder.setVolumeName(volumeName)
@@ -546,7 +582,8 @@
@Override
public void deleteBucket(
String volumeName, String bucketName) throws IOException {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
+ verifyVolumeName(volumeName);
+ verifyBucketName(bucketName);
ozoneManagerClient.deleteBucket(volumeName, bucketName);
}
@@ -559,7 +596,8 @@
@Override
public OzoneBucket getBucketDetails(
String volumeName, String bucketName) throws IOException {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
+ verifyVolumeName(volumeName);
+ verifyBucketName(bucketName);
OmBucketInfo bucketInfo =
ozoneManagerClient.getBucketInfo(volumeName, bucketName);
return new OzoneBucket(
@@ -602,7 +640,8 @@
ReplicationType type, ReplicationFactor factor,
Map<String, String> metadata)
throws IOException {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
+ verifyVolumeName(volumeName);
+ verifyBucketName(bucketName);
HddsClientUtils.checkNotNull(keyName, type, factor);
String requestId = UUID.randomUUID().toString();
@@ -649,7 +688,8 @@
public OzoneInputStream getKey(
String volumeName, String bucketName, String keyName)
throws IOException {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
+ verifyVolumeName(volumeName);
+ verifyBucketName(bucketName);
Preconditions.checkNotNull(keyName);
OmKeyArgs keyArgs = new OmKeyArgs.Builder()
.setVolumeName(volumeName)
@@ -666,7 +706,8 @@
public void deleteKey(
String volumeName, String bucketName, String keyName)
throws IOException {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
+ verifyVolumeName(volumeName);
+ verifyBucketName(bucketName);
Preconditions.checkNotNull(keyName);
OmKeyArgs keyArgs = new OmKeyArgs.Builder()
.setVolumeName(volumeName)
@@ -679,7 +720,8 @@
@Override
public void renameKey(String volumeName, String bucketName,
String fromKeyName, String toKeyName) throws IOException {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
+ verifyVolumeName(volumeName);
+ verifyBucketName(bucketName);
HddsClientUtils.checkNotNull(fromKeyName, toKeyName);
OmKeyArgs keyArgs = new OmKeyArgs.Builder()
.setVolumeName(volumeName)
@@ -756,74 +798,6 @@
}
@Override
- public void createS3Bucket(String userName, String s3BucketName)
- throws IOException {
- Preconditions.checkArgument(Strings.isNotBlank(userName), "user name " +
- "cannot be null or empty.");
-
- Preconditions.checkArgument(Strings.isNotBlank(s3BucketName), "bucket " +
- "name cannot be null or empty.");
- try {
- HddsClientUtils.verifyResourceName(s3BucketName);
- } catch (IllegalArgumentException exception) {
- throw new OMException("Invalid bucket name: " + s3BucketName,
- OMException.ResultCodes.INVALID_BUCKET_NAME);
- }
- ozoneManagerClient.createS3Bucket(userName, s3BucketName);
- }
-
- @Override
- public void deleteS3Bucket(String s3BucketName)
- throws IOException {
- Preconditions.checkArgument(Strings.isNotBlank(s3BucketName), "bucket " +
- "name cannot be null or empty.");
- ozoneManagerClient.deleteS3Bucket(s3BucketName);
- }
-
- @Override
- public String getOzoneBucketMapping(String s3BucketName) throws IOException {
- Preconditions.checkArgument(Strings.isNotBlank(s3BucketName), "bucket " +
- "name cannot be null or empty.");
- return ozoneManagerClient.getOzoneBucketMapping(s3BucketName);
- }
-
- @Override
- @SuppressWarnings("StringSplitter")
- public String getOzoneVolumeName(String s3BucketName) throws IOException {
- String mapping = getOzoneBucketMapping(s3BucketName);
- return mapping.split("/")[0];
-
- }
-
- @Override
- @SuppressWarnings("StringSplitter")
- public String getOzoneBucketName(String s3BucketName) throws IOException {
- String mapping = getOzoneBucketMapping(s3BucketName);
- return mapping.split("/")[1];
- }
-
- @Override
- public List<OzoneBucket> listS3Buckets(String userName, String bucketPrefix,
- String prevBucket, int maxListResult)
- throws IOException {
- List<OmBucketInfo> buckets = ozoneManagerClient.listS3Buckets(
- userName, prevBucket, bucketPrefix, maxListResult);
-
- return buckets.stream().map(bucket -> new OzoneBucket(
- conf,
- this,
- bucket.getVolumeName(),
- bucket.getBucketName(),
- bucket.getStorageType(),
- bucket.getIsVersionEnabled(),
- bucket.getCreationTime(),
- bucket.getMetadata(),
- bucket.getEncryptionKeyInfo() != null ?
- bucket.getEncryptionKeyInfo().getKeyName(): null))
- .collect(Collectors.toList());
- }
-
- @Override
public void close() throws IOException {
IOUtils.cleanupWithLogger(LOG, ozoneManagerClient);
IOUtils.cleanupWithLogger(LOG, xceiverClientManager);
@@ -836,7 +810,8 @@
ReplicationType type,
ReplicationFactor factor)
throws IOException {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
+ verifyVolumeName(volumeName);
+ verifyBucketName(bucketName);
HddsClientUtils.checkNotNull(keyName, type, factor);
OmKeyArgs keyArgs = new OmKeyArgs.Builder()
.setVolumeName(volumeName)
@@ -859,7 +834,8 @@
int partNumber,
String uploadID)
throws IOException {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
+ verifyVolumeName(volumeName);
+ verifyBucketName(bucketName);
HddsClientUtils.checkNotNull(keyName, uploadID);
Preconditions.checkArgument(partNumber > 0 && partNumber <=10000, "Part " +
"number should be greater than zero and less than or equal to 10000");
@@ -909,7 +885,8 @@
public OmMultipartUploadCompleteInfo completeMultipartUpload(
String volumeName, String bucketName, String keyName, String uploadID,
Map<Integer, String> partsMap) throws IOException {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
+ verifyVolumeName(volumeName);
+ verifyBucketName(bucketName);
HddsClientUtils.checkNotNull(keyName, uploadID);
OmKeyArgs keyArgs = new OmKeyArgs.Builder()
@@ -935,7 +912,8 @@
@Override
public void abortMultipartUpload(String volumeName,
String bucketName, String keyName, String uploadID) throws IOException {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
+ verifyVolumeName(volumeName);
+ verifyBucketName(bucketName);
HddsClientUtils.checkNotNull(keyName, uploadID);
OmKeyArgs omKeyArgs = new OmKeyArgs.Builder()
.setVolumeName(volumeName)
@@ -950,7 +928,8 @@
public OzoneMultipartUploadPartListParts listParts(String volumeName,
String bucketName, String keyName, String uploadID, int partNumberMarker,
int maxParts) throws IOException {
- HddsClientUtils.verifyResourceName(volumeName, bucketName);
+ verifyVolumeName(volumeName);
+ verifyBucketName(bucketName);
HddsClientUtils.checkNotNull(uploadID);
Preconditions.checkArgument(maxParts > 0, "Max Parts Should be greater " +
"than zero");
@@ -1192,6 +1171,7 @@
.setFactor(HddsProtos.ReplicationFactor.valueOf(factor.getValue()))
.setStreamBufferSize(streamBufferSize)
.setStreamBufferFlushSize(streamBufferFlushSize)
+ .setStreamBufferFlushDelay(streamBufferFlushDelay)
.setStreamBufferMaxSize(streamBufferMaxSize)
.setBlockSize(blockSize)
.setChecksumType(checksumType)
diff --git a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java
index b612383..8f8659d 100644
--- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java
+++ b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java
@@ -18,23 +18,18 @@
package org.apache.hadoop.ozone.client;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.OMConfigKeys;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.junit.rules.Timeout;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.List;
-
+import org.apache.commons.lang3.StringUtils;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT;
@@ -43,6 +38,10 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.rules.Timeout;
/**
* This test class verifies the parsing of SCM endpoint config settings. The
@@ -61,7 +60,7 @@
*/
@Test
public void testMissingScmClientAddress() {
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
thrown.expect(IllegalArgumentException.class);
HddsUtils.getScmAddressForClients(conf);
}
@@ -72,7 +71,7 @@
*/
@Test
public void testGetScmClientAddress() {
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
// First try a client address with just a host name. Verify it falls
// back to the default port.
@@ -91,7 +90,7 @@
@Test
public void testgetOmSocketAddress() {
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
// First try a client address with just a host name. Verify it falls
// back to the default port.
@@ -119,7 +118,7 @@
// When OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY is undefined it should
// fallback to OZONE_SCM_CLIENT_ADDRESS_KEY.
final String scmHost = "host123";
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, scmHost);
final InetSocketAddress address = HddsUtils.getScmAddressForBlockClients(
conf);
@@ -136,7 +135,7 @@
// Verify that the OZONE_SCM_CLIENT_ADDRESS_KEY port number is ignored,
// if present. Instead we should use OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT.
final String scmHost = "host123:100";
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, scmHost);
final InetSocketAddress address =HddsUtils.getScmAddressForBlockClients(
conf);
@@ -149,7 +148,7 @@
// When OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY and OZONE_SCM_CLIENT_ADDRESS_KEY
// are undefined it should fallback to OZONE_SCM_NAMES.
final String scmHost = "host456";
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
conf.set(OZONE_SCM_NAMES, scmHost);
final InetSocketAddress address = HddsUtils.getScmAddressForBlockClients(
conf);
@@ -166,7 +165,7 @@
// Verify that the OZONE_SCM_NAMES port number is ignored, if present.
// Instead we should use OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT.
final String scmHost = "host456:200";
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
conf.set(OZONE_SCM_NAMES, scmHost);
final InetSocketAddress address = HddsUtils.getScmAddressForBlockClients(
conf);
@@ -179,7 +178,7 @@
// When OZONE_SCM_CLIENT_ADDRESS_KEY is undefined, it should fallback
// to OZONE_SCM_NAMES.
final String scmHost = "host456";
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
conf.set(OZONE_SCM_NAMES, scmHost);
final InetSocketAddress address = HddsUtils.getScmAddressForClients(conf);
assertEquals(scmHost, address.getHostName());
@@ -195,7 +194,7 @@
// Verify that the OZONE_SCM_NAMES port number is ignored, if present.
// Instead we should use OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT.
final String scmHost = "host456:300";
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
conf.set(OZONE_SCM_NAMES, scmHost);
final InetSocketAddress address = HddsUtils.getScmAddressForClients(conf);
assertEquals(scmHost.split(":")[0], address.getHostName());
@@ -207,7 +206,7 @@
// When OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY and OZONE_SCM_CLIENT_ADDRESS_KEY
// are undefined, fail if OZONE_SCM_NAMES has multiple SCMs.
final String scmHost = "host123,host456";
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
conf.set(OZONE_SCM_NAMES, scmHost);
thrown.expect(IllegalArgumentException.class);
HddsUtils.getScmAddressForBlockClients(conf);
@@ -218,7 +217,7 @@
// When OZONE_SCM_CLIENT_ADDRESS_KEY is undefined, fail if OZONE_SCM_NAMES
// has multiple SCMs.
final String scmHost = "host123,host456";
- final Configuration conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
conf.set(OZONE_SCM_NAMES, scmHost);
thrown.expect(IllegalArgumentException.class);
HddsUtils.getScmAddressForClients(conf);
diff --git a/hadoop-ozone/common/pom.xml b/hadoop-ozone/common/pom.xml
index ed85afa..0c02085 100644
--- a/hadoop-ozone/common/pom.xml
+++ b/hadoop-ozone/common/pom.xml
@@ -36,21 +36,8 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs-client</artifactId>
- <exclusions>
- <exclusion>
- <groupId>com.squareup.okhttp</groupId>
- <artifactId>okhttp</artifactId>
- </exclusion>
- </exclusions>
+ <artifactId>hadoop-hdds-test-utils</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -61,19 +48,14 @@
<artifactId>hadoop-hdds-client</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdds-tools</artifactId>
- </dependency>
- <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
+ <artifactId>hadoop-hdds-hadoop-dependency-test</artifactId>
<scope>test</scope>
- <type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -102,6 +84,36 @@
</resources>
<plugins>
<plugin>
+ <groupId>com.salesforce.servicelibs</groupId>
+ <artifactId>proto-backwards-compatibility</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.xolstice.maven.plugins</groupId>
+ <artifactId>protobuf-maven-plugin</artifactId>
+ <version>${protobuf-maven-plugin.version}</version>
+ <extensions>true</extensions>
+ <executions>
+ <execution>
+ <id>compile-protoc</id>
+ <goals>
+ <goal>compile</goal>
+ <goal>test-compile</goal>
+ </goals>
+ <configuration>
+ <additionalProtoPathElements>
+ <param>
+ ${basedir}/../../hadoop-hdds/common/src/main/proto/
+ </param>
+ <param>${basedir}/src/main/proto</param>
+ </additionalProtoPathElements>
+ <protocArtifact>
+ com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}
+ </protocArtifact>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-maven-plugins</artifactId>
<executions>
@@ -121,28 +133,6 @@
</source>
</configuration>
</execution>
- <execution>
- <id>compile-protoc</id>
- <goals>
- <goal>protoc</goal>
- </goals>
- <configuration>
- <protocVersion>${protobuf.version}</protocVersion>
- <protocCommand>${protoc.path}</protocCommand>
- <imports>
- <param>
- ${basedir}/../../hadoop-hdds/common/src/main/proto/
- </param>
- <param>${basedir}/src/main/proto</param>
- </imports>
- <source>
- <directory>${basedir}/src/main/proto</directory>
- <includes>
- <include>OzoneManagerProtocol.proto</include>
- </includes>
- </source>
- </configuration>
- </execution>
</executions>
</plugin>
<plugin>
@@ -162,7 +152,6 @@
<plugin>
<groupId>io.fabric8</groupId>
<artifactId>docker-maven-plugin</artifactId>
- <version>0.29.0</version>
<configuration>
<images>
<image>
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
index 3552e79..12220cd 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@ -17,15 +17,10 @@
package org.apache.hadoop.ozone;
-import com.google.common.base.Joiner;
import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
-import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
@@ -37,24 +32,20 @@
import java.util.Map;
import java.util.Optional;
import java.util.OptionalInt;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import org.apache.commons.compress.archivers.ArchiveEntry;
-import org.apache.commons.compress.archivers.ArchiveOutputStream;
-import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
-import org.apache.commons.compress.compressors.CompressorException;
-import org.apache.commons.compress.compressors.CompressorOutputStream;
-import org.apache.commons.compress.compressors.CompressorStreamFactory;
-import org.apache.commons.compress.utils.IOUtils;
-import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.ozone.conf.OMClientConfig;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import com.google.common.base.Joiner;
+import org.apache.commons.lang3.StringUtils;
import static org.apache.hadoop.hdds.HddsUtils.getHostNameFromConfigKeys;
import static org.apache.hadoop.hdds.HddsUtils.getPortNumberFromConfigKeys;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
@@ -68,7 +59,6 @@
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_NODES_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_PORT_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -89,18 +79,18 @@
* @param conf
* @return Target InetSocketAddress for the SCM service endpoint.
*/
- public static InetSocketAddress getOmAddress(Configuration conf) {
+ public static InetSocketAddress getOmAddress(ConfigurationSource conf) {
return NetUtils.createSocketAddr(getOmRpcAddress(conf));
}
/**
* Return list of OM addresses by service ids - when HA is enabled.
*
- * @param conf {@link Configuration}
+ * @param conf {@link ConfigurationSource}
* @return {service.id -> [{@link InetSocketAddress}]}
*/
public static Map<String, List<InetSocketAddress>> getOmHAAddressesById(
- Configuration conf) {
+ ConfigurationSource conf) {
Map<String, List<InetSocketAddress>> result = new HashMap<>();
for (String serviceId : conf.getTrimmedStringCollection(
OZONE_OM_SERVICE_IDS_KEY)) {
@@ -126,7 +116,7 @@
* @param conf
* @return Target InetSocketAddress for the SCM service endpoint.
*/
- public static String getOmRpcAddress(Configuration conf) {
+ public static String getOmRpcAddress(ConfigurationSource conf) {
final Optional<String> host = getHostNameFromConfigKeys(conf,
OZONE_OM_ADDRESS_KEY);
@@ -141,7 +131,8 @@
* @param confKey configuration key to lookup address from
* @return Target InetSocketAddress for the OM RPC server.
*/
- public static String getOmRpcAddress(Configuration conf, String confKey) {
+ public static String getOmRpcAddress(ConfigurationSource conf,
+ String confKey) {
final Optional<String> host = getHostNameFromConfigKeys(conf, confKey);
if (host.isPresent()) {
@@ -159,7 +150,7 @@
* @return Target InetSocketAddress for the OM service endpoint.
*/
public static InetSocketAddress getOmAddressForClients(
- Configuration conf) {
+ ConfigurationSource conf) {
final Optional<String> host = getHostNameFromConfigKeys(conf,
OZONE_OM_ADDRESS_KEY);
@@ -180,7 +171,7 @@
* @return true if OZONE_OM_SERVICE_IDS_KEY is defined and not empty;
* else false.
*/
- public static boolean isServiceIdsDefined(Configuration conf) {
+ public static boolean isServiceIdsDefined(ConfigurationSource conf) {
String val = conf.get(OZONE_OM_SERVICE_IDS_KEY);
return val != null && val.length() > 0;
}
@@ -191,13 +182,14 @@
* @param serviceId OM HA cluster service ID
* @return true if HA is configured in the configuration; else false.
*/
- public static boolean isOmHAServiceId(Configuration conf, String serviceId) {
+ public static boolean isOmHAServiceId(ConfigurationSource conf,
+ String serviceId) {
Collection<String> omServiceIds = conf.getTrimmedStringCollection(
OZONE_OM_SERVICE_IDS_KEY);
return omServiceIds.contains(serviceId);
}
- public static int getOmRpcPort(Configuration conf) {
+ public static int getOmRpcPort(ConfigurationSource conf) {
return getPortNumberFromConfigKeys(conf, OZONE_OM_ADDRESS_KEY)
.orElse(OZONE_OM_PORT_DEFAULT);
}
@@ -209,12 +201,12 @@
* @param confKey configuration key to lookup address from
* @return Port on which OM RPC server will listen on
*/
- public static int getOmRpcPort(Configuration conf, String confKey) {
+ public static int getOmRpcPort(ConfigurationSource conf, String confKey) {
return getPortNumberFromConfigKeys(conf, confKey)
.orElse(OZONE_OM_PORT_DEFAULT);
}
- public static int getOmRestPort(Configuration conf) {
+ public static int getOmRestPort(ConfigurationSource conf) {
return getPortNumberFromConfigKeys(conf, OZONE_OM_HTTP_ADDRESS_KEY)
.orElse(OZONE_OM_HTTP_BIND_PORT_DEFAULT);
}
@@ -237,9 +229,6 @@
case LookupKey:
case ListKeys:
case ListTrash:
- case RecoverTrash:
- case InfoS3Bucket:
- case ListS3Buckets:
case ServiceList:
case ListMultiPartUploadParts:
case GetFileStatus:
@@ -260,8 +249,6 @@
case DeleteKey:
case CommitKey:
case AllocateBlock:
- case CreateS3Bucket:
- case DeleteS3Bucket:
case InitiateMultiPartUpload:
case CommitMultiPartUpload:
case CompleteMultiPartUpload:
@@ -276,6 +263,7 @@
case SetAcl:
case AddAcl:
case PurgeKeys:
+ case RecoverTrash:
return false;
default:
LOG.error("CmdType {} is not categorized as readOnly or not.", cmdType);
@@ -345,7 +333,7 @@
/**
* Get a collection of all omNodeIds for the given omServiceId.
*/
- public static Collection<String> getOMNodeIds(Configuration conf,
+ public static Collection<String> getOMNodeIds(ConfigurationSource conf,
String omServiceId) {
String key = addSuffix(OZONE_OM_NODES_KEY, omServiceId);
return conf.getTrimmedStringCollection(key);
@@ -364,54 +352,7 @@
}
}
- /**
- * Write OM DB Checkpoint to an output stream as a compressed file (tgz).
- * @param checkpoint checkpoint file
- * @param destination desination output stream.
- * @throws IOException
- */
- public static void writeOmDBCheckpointToStream(DBCheckpoint checkpoint,
- OutputStream destination)
- throws IOException {
- try (CompressorOutputStream gzippedOut = new CompressorStreamFactory()
- .createCompressorOutputStream(CompressorStreamFactory.GZIP,
- destination)) {
-
- try (ArchiveOutputStream archiveOutputStream =
- new TarArchiveOutputStream(gzippedOut)) {
-
- Path checkpointPath = checkpoint.getCheckpointLocation();
- try (Stream<Path> files = Files.list(checkpointPath)) {
- for (Path path : files.collect(Collectors.toList())) {
- if (path != null) {
- Path fileName = path.getFileName();
- if (fileName != null) {
- includeFile(path.toFile(), fileName.toString(),
- archiveOutputStream);
- }
- }
- }
- }
- }
- } catch (CompressorException e) {
- throw new IOException(
- "Can't compress the checkpoint: " +
- checkpoint.getCheckpointLocation(), e);
- }
- }
-
- private static void includeFile(File file, String entryName,
- ArchiveOutputStream archiveOutputStream)
- throws IOException {
- ArchiveEntry archiveEntry =
- archiveOutputStream.createArchiveEntry(file, entryName);
- archiveOutputStream.putArchiveEntry(archiveEntry);
- try (FileInputStream fis = new FileInputStream(file)) {
- IOUtils.copy(fis, archiveOutputStream);
- }
- archiveOutputStream.closeArchiveEntry();
- }
/**
* If a OM conf is only set with key suffixed with OM Node ID, return the
@@ -419,7 +360,7 @@
* @return if the value is set for key suffixed with OM Node ID, return the
* value, else return null.
*/
- public static String getConfSuffixedWithOMNodeId(Configuration conf,
+ public static String getConfSuffixedWithOMNodeId(ConfigurationSource conf,
String confKey, String omServiceID, String omNodeId) {
String suffixedConfKey = OmUtils.addKeySuffixes(
confKey, omServiceID, omNodeId);
@@ -437,7 +378,7 @@
* @param omNodeHostAddr peer OM node host address
* @return http address of peer OM node in the format <hostName>:<port>
*/
- public static String getHttpAddressForOMPeerNode(Configuration conf,
+ public static String getHttpAddressForOMPeerNode(ConfigurationSource conf,
String omServiceId, String omNodeId, String omNodeHostAddr) {
final Optional<String> bindHost = getHostNameFromConfigKeys(conf,
addKeySuffixes(OZONE_OM_HTTP_BIND_HOST_KEY, omServiceId, omNodeId));
@@ -460,7 +401,7 @@
* @param omNodeHostAddr peer OM node host address
* @return https address of peer OM node in the format <hostName>:<port>
*/
- public static String getHttpsAddressForOMPeerNode(Configuration conf,
+ public static String getHttpsAddressForOMPeerNode(ConfigurationSource conf,
String omServiceId, String omNodeId, String omNodeHostAddr) {
final Optional<String> bindHost = getHostNameFromConfigKeys(conf,
addKeySuffixes(OZONE_OM_HTTPS_BIND_HOST_KEY, omServiceId, omNodeId));
@@ -528,4 +469,51 @@
return repeatedOmKeyInfo;
}
+
+ /**
+ * Verify volume name is a valid DNS name.
+ */
+ public static void validateVolumeName(String volumeName) throws OMException {
+ try {
+ HddsClientUtils.verifyResourceName(volumeName);
+ } catch (IllegalArgumentException e) {
+ throw new OMException("Invalid volume name: " + volumeName,
+ OMException.ResultCodes.INVALID_VOLUME_NAME);
+ }
+ }
+
+ /**
+ * Verify bucket name is a valid DNS name.
+ */
+ public static void validateBucketName(String bucketName)
+ throws OMException {
+ try {
+ HddsClientUtils.verifyResourceName(bucketName);
+ } catch (IllegalArgumentException e) {
+ throw new OMException("Invalid bucket name: " + bucketName,
+ OMException.ResultCodes.INVALID_BUCKET_NAME);
+ }
+ }
+
+ /**
+ * Return OM Client Rpc Time out.
+ */
+ public static long getOMClientRpcTimeOut(Configuration configuration) {
+ return OzoneConfiguration.of(configuration)
+ .getObject(OMClientConfig.class).getRpcTimeOut();
+ }
+
+ /**
+ * Return OmKeyInfo that would be recovered.
+ */
+ public static OmKeyInfo prepareKeyForRecover(OmKeyInfo keyInfo,
+ RepeatedOmKeyInfo repeatedOmKeyInfo) {
+
+ /* TODO: HDDS-2425. HDDS-2426.*/
+ if (repeatedOmKeyInfo.getOmKeyInfoList().contains(keyInfo)) {
+ return keyInfo;
+ } else {
+ return null;
+ }
+ }
}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/conf/OMClientConfig.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/conf/OMClientConfig.java
new file mode 100644
index 0000000..37cd67e
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/conf/OMClientConfig.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.ozone.conf;
+
+import org.apache.hadoop.hdds.conf.Config;
+import org.apache.hadoop.hdds.conf.ConfigGroup;
+import org.apache.hadoop.hdds.conf.ConfigType;
+
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.hadoop.hdds.conf.ConfigTag.CLIENT;
+import static org.apache.hadoop.hdds.conf.ConfigTag.OM;
+import static org.apache.hadoop.hdds.conf.ConfigTag.OZONE;
+
+/**
+ * Config for OM Client.
+ */
+@ConfigGroup(prefix = "ozone.om.client")
+public class OMClientConfig {
+
+ public static final String OM_CLIENT_RPC_TIME_OUT = "rpc.timeout";
+
+ @Config(key = OM_CLIENT_RPC_TIME_OUT,
+ defaultValue = "15m",
+ type = ConfigType.TIME,
+ tags = {OZONE, OM, CLIENT},
+ timeUnit = TimeUnit.MILLISECONDS,
+ description = "RpcClient timeout on waiting for the response from " +
+ "OzoneManager. The default value is set to 15 minutes. " +
+ "If ipc.client.ping is set to true and this rpc-timeout " +
+ "is greater than the value of ipc.ping.interval, the effective " +
+ "value of the rpc-timeout is rounded up to multiple of " +
+ "ipc.ping.interval."
+ )
+ private long rpcTimeOut = 15 * 60 * 1000;
+
+
+ public long getRpcTimeOut() {
+ return rpcTimeOut;
+ }
+
+ public void setRpcTimeOut(long timeOut) {
+ // As at the end this value should not exceed MAX_VALUE, as underlying
+ // Rpc layer SocketTimeout parameter is int.
+ if (rpcTimeOut > Integer.MAX_VALUE) {
+ this.rpcTimeOut = Integer.MAX_VALUE;
+ }
+ this.rpcTimeOut = timeOut;
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/bucket/package-info.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/conf/package-info.java
similarity index 87%
copy from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/bucket/package-info.java
copy to hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/conf/package-info.java
index f484ecc..3e7ec6a 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/bucket/package-info.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/conf/package-info.java
@@ -18,7 +18,5 @@
*/
/**
- * Package contains classes related to s3 bucket responses.
- */
-package org.apache.hadoop.ozone.om.response.s3.bucket;
-
+ * Package contains classes related to ozone configuration.
+ */
\ No newline at end of file
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/freon/OzoneGetConf.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/freon/OzoneGetConf.java
index 83283d4..e14fe3f 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/freon/OzoneGetConf.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/freon/OzoneGetConf.java
@@ -25,13 +25,13 @@
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
+
import org.apache.hadoop.HadoopIllegalArgumentException;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdds.HddsUtils;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.util.StringUtils;
@@ -98,8 +98,6 @@
static final String USAGE;
static {
- HdfsConfiguration.init();
-
/* Initialize USAGE based on Command values */
StringBuilder usage = new StringBuilder(DESCRIPTION);
usage.append("\nozone getconf \n");
@@ -178,11 +176,12 @@
private final PrintStream out; // Stream for printing command output
private final PrintStream err; // Stream for printing error
- protected OzoneGetConf(Configuration conf) {
+ protected OzoneGetConf(OzoneConfiguration conf) {
this(conf, System.out, System.err);
}
- protected OzoneGetConf(Configuration conf, PrintStream out, PrintStream err) {
+ protected OzoneGetConf(OzoneConfiguration conf, PrintStream out,
+ PrintStream err) {
super(conf);
this.out = out;
this.err = err;
@@ -237,7 +236,7 @@
public int doWorkInternal(OzoneGetConf tool, String[] args)
throws IOException {
Collection<InetSocketAddress> addresses = HddsUtils
- .getSCMAddresses(tool.getConf());
+ .getSCMAddresses(OzoneConfiguration.of(tool.getConf()));
for (InetSocketAddress addr : addresses) {
tool.printOut(addr.getHostName());
@@ -253,11 +252,15 @@
@Override
public int doWorkInternal(OzoneGetConf tool, String[] args)
throws IOException {
- if (OmUtils.isServiceIdsDefined(tool.getConf())) {
- tool.printOut(OmUtils.getOmHAAddressesById(tool.getConf()).toString());
+ ConfigurationSource configSource =
+ OzoneConfiguration.of(tool.getConf());
+ if (OmUtils.isServiceIdsDefined(
+ configSource)) {
+ tool.printOut(OmUtils.getOmHAAddressesById(configSource).toString());
} else {
- tool.printOut(OmUtils.getOmAddress(tool.getConf()).getHostName());
+ tool.printOut(OmUtils.getOmAddress(configSource).getHostName());
}
+
return 0;
}
}
@@ -267,7 +270,7 @@
System.exit(0);
}
- Configuration conf = new Configuration();
+ OzoneConfiguration conf = new OzoneConfiguration();
conf.addResource(new OzoneConfiguration());
int res = ToolRunner.run(new OzoneGetConf(conf), args);
System.exit(res);
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index f46b308..7800d2f 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@ -76,6 +76,9 @@
"ozone.om.db.cache.size.mb";
public static final int OZONE_OM_DB_CACHE_SIZE_DEFAULT = 128;
+ public static final String OZONE_OM_VOLUME_LISTALL_ALLOWED =
+ "ozone.om.volume.listall.allowed";
+ public static final boolean OZONE_OM_VOLUME_LISTALL_ALLOWED_DEFAULT = true;
public static final String OZONE_OM_USER_MAX_VOLUME =
"ozone.om.user.max.volume";
public static final int OZONE_OM_USER_MAX_VOLUME_DEFAULT = 1024;
@@ -200,9 +203,14 @@
public static final String OZONE_OM_KERBEROS_PRINCIPAL_KEY = "ozone.om"
+ ".kerberos.principal";
public static final String OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE =
- "ozone.om.http.kerberos.keytab";
+ "ozone.om.http.auth.kerberos.keytab";
public static final String OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY
- = "ozone.om.http.kerberos.principal";
+ = "ozone.om.http.auth.kerberos.principal";
+ public static final String OZONE_OM_HTTP_AUTH_TYPE =
+ "ozone.om.http.auth.type";
+ public static final String OZONE_OM_HTTP_AUTH_CONFIG_PREFIX =
+ "ozone.om.http.auth.";
+
// Delegation token related keys
public static final String DELEGATION_REMOVER_SCAN_INTERVAL_KEY =
"ozone.manager.delegation.remover.scan.interval";
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
index 279fda7..58d5a02 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
@@ -145,10 +145,6 @@
SCM_VERSION_MISMATCH_ERROR,
- S3_BUCKET_NOT_FOUND,
-
- S3_BUCKET_ALREADY_EXISTS,
-
INITIATE_MULTIPART_UPLOAD_ERROR,
MULTIPART_UPLOAD_PARTFILE_ERROR,
@@ -203,8 +199,6 @@
PREFIX_NOT_FOUND,
- S3_BUCKET_INVALID_LENGTH,
-
RATIS_ERROR, // Error in Ratis server
INVALID_PATH_IN_ACL_REQUEST, // Error code when path name is invalid during
@@ -227,6 +221,8 @@
DIRECTORY_ALREADY_EXISTS,
+ INVALID_VOLUME_NAME,
+
REPLAY // When ratis logs are replayed.
}
}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/ha/OMFailoverProxyProvider.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/ha/OMFailoverProxyProvider.java
index 37a637e..b0991a6 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/ha/OMFailoverProxyProvider.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/ha/OMFailoverProxyProvider.java
@@ -18,24 +18,6 @@
package org.apache.hadoop.ozone.om.ha;
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.retry.FailoverProxyProvider;
-import org.apache.hadoop.io.retry.RetryInvocationHandler;
-import org.apache.hadoop.ipc.Client;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.ozone.OmUtils;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB;
-import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.Closeable;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -43,10 +25,32 @@
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.retry.FailoverProxyProvider;
+import org.apache.hadoop.io.retry.RetryInvocationHandler;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB;
+import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB;
+import org.apache.hadoop.security.UserGroupInformation;
+
+import com.google.common.annotations.VisibleForTesting;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A failover proxy provider implementation which allows clients to configure
@@ -67,14 +71,25 @@
private String currentProxyOMNodeId;
private int currentProxyIndex;
- private final Configuration conf;
+ private final ConfigurationSource conf;
private final long omVersion;
private final UserGroupInformation ugi;
private final Text delegationTokenService;
private final String omServiceId;
- public OMFailoverProxyProvider(OzoneConfiguration configuration,
+
+ // OMFailoverProxyProvider, on encountering certain exception, tries each OM
+ // once in a round robin fashion. After that it waits for configured time
+ // before attempting to contact all the OMs again. For other exceptions
+ // such as LeaderNotReadyException, the same OM is contacted again with a
+ // linearly increasing wait time.
+ private Set<String> attemptedOMs = new HashSet<>();
+ private String lastAttemptedOM;
+ private int numAttemptsOnSameOM = 0;
+ private final long waitBetweenRetries;
+
+ public OMFailoverProxyProvider(ConfigurationSource configuration,
UserGroupInformation ugi, String omServiceId) throws IOException {
this.conf = configuration;
this.omVersion = RPC.getProtocolVersion(OzoneManagerProtocolPB.class);
@@ -85,6 +100,10 @@
currentProxyIndex = 0;
currentProxyOMNodeId = omNodeIDList.get(currentProxyIndex);
+
+ waitBetweenRetries = conf.getLong(
+ OzoneConfigKeys.OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_KEY,
+ OzoneConfigKeys.OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_DEFAULT);
}
public OMFailoverProxyProvider(OzoneConfiguration configuration,
@@ -92,7 +111,7 @@
this(configuration, ugi, null);
}
- private void loadOMClientConfigs(Configuration config, String omSvcId)
+ private void loadOMClientConfigs(ConfigurationSource config, String omSvcId)
throws IOException {
this.omProxies = new HashMap<>();
this.omProxyInfos = new HashMap<>();
@@ -149,11 +168,14 @@
private OzoneManagerProtocolPB createOMProxy(InetSocketAddress omAddress)
throws IOException {
- RPC.setProtocolEngine(conf, OzoneManagerProtocolPB.class,
+ Configuration hadoopConf =
+ LegacyHadoopConfigurationSource.asHadoopConfiguration(conf);
+ RPC.setProtocolEngine(hadoopConf, OzoneManagerProtocolPB.class,
ProtobufRpcEngine.class);
return RPC.getProxy(OzoneManagerProtocolPB.class, omVersion, omAddress, ugi,
- conf, NetUtils.getDefaultSocketFactory(conf),
- Client.getRpcTimeout(conf));
+ hadoopConf, NetUtils.getDefaultSocketFactory(hadoopConf),
+ (int) OmUtils.getOMClientRpcTimeOut(hadoopConf));
+
}
/**
@@ -176,7 +198,13 @@
if (proxyInfo.proxy == null) {
InetSocketAddress address = omProxyInfos.get(nodeId).getAddress();
try {
- proxyInfo.proxy = createOMProxy(address);
+ OzoneManagerProtocolPB proxy = createOMProxy(address);
+ try {
+ proxyInfo.proxy = proxy;
+ } catch (IllegalAccessError iae) {
+ omProxies.put(nodeId,
+ new ProxyInfo<>(proxy, proxyInfo.proxyInfo));
+ }
} catch (IOException ioe) {
LOG.error("{} Failed to create RPC proxy to OM at {}",
this.getClass().getSimpleName(), address, ioe);
@@ -191,7 +219,7 @@
private Text computeDelegationTokenService() {
// For HA, this will return "," separated address of all OM's.
- StringBuilder rpcAddress = new StringBuilder();
+ List<String> addresses = new ArrayList<>();
for (Map.Entry<String, OMProxyInfo> omProxyInfoSet :
omProxyInfos.entrySet()) {
@@ -200,11 +228,18 @@
// During client object creation when one of the OM configured address
// in unreachable, dtService can be null.
if (dtService != null) {
- rpcAddress.append(",").append(dtService);
+ addresses.add(dtService.toString());
}
}
- return new Text(rpcAddress.toString().substring(1));
+ if (!addresses.isEmpty()) {
+ Collections.sort(addresses);
+ return new Text(String.join(",", addresses));
+ } else {
+ // If all OM addresses are unresolvable, set dt service to null. Let
+ // this fail in later step when during connection setup.
+ return null;
+ }
}
@Override
@@ -255,7 +290,7 @@
}
/**
- * Performs failover if the leaderOMNodeId returned through OMReponse does
+ * Performs failover if the leaderOMNodeId returned through OMResponse does
* not match the current leaderOMNodeId cached by the proxy provider.
*/
public void performFailoverToNextProxy() {
@@ -271,6 +306,11 @@
* @return the new proxy index
*/
private synchronized int incrementProxyIndex() {
+ // Before failing over to next proxy, add the proxy OM (which has
+ // returned an exception) to the list of attemptedOMs.
+ lastAttemptedOM = currentProxyOMNodeId;
+ attemptedOMs.add(currentProxyOMNodeId);
+
currentProxyIndex = (currentProxyIndex + 1) % omProxies.size();
currentProxyOMNodeId = omNodeIDList.get(currentProxyIndex);
return currentProxyIndex;
@@ -284,10 +324,13 @@
synchronized boolean updateLeaderOMNodeId(String newLeaderOMNodeId) {
if (!currentProxyOMNodeId.equals(newLeaderOMNodeId)) {
if (omProxies.containsKey(newLeaderOMNodeId)) {
+ lastAttemptedOM = currentProxyOMNodeId;
currentProxyOMNodeId = newLeaderOMNodeId;
currentProxyIndex = omNodeIDList.indexOf(currentProxyOMNodeId);
return true;
}
+ } else {
+ lastAttemptedOM = currentProxyOMNodeId;
}
return false;
}
@@ -296,6 +339,32 @@
return currentProxyIndex;
}
+ public synchronized long getWaitTime() {
+ if (currentProxyOMNodeId.equals(lastAttemptedOM)) {
+ // Clear attemptedOMs list as round robin has been broken. Add only the
+ attemptedOMs.clear();
+
+ // The same OM will be contacted again. So wait and then retry.
+ numAttemptsOnSameOM++;
+ return (waitBetweenRetries * numAttemptsOnSameOM);
+ }
+ // Reset numAttemptsOnSameOM as we failed over to a different OM.
+ numAttemptsOnSameOM = 0;
+
+ // OMs are being contacted in round robin way. Check if all the OMs have
+ // been contacted in this attempt.
+ for (String omNodeID : omProxyInfos.keySet()) {
+ if (!attemptedOMs.contains(omNodeID)) {
+ return 0;
+ }
+ }
+ // This implies all the OMs have been contacted once. Return true and
+ // clear the list as we are going to inject a wait and the next check
+ // should not include these atttempts again.
+ attemptedOMs.clear();
+ return waitBetweenRetries;
+ }
+
/**
* Close all the proxy objects which have been opened over the lifetime of
* the proxy provider.
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/DBUpdates.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/DBUpdates.java
new file mode 100644
index 0000000..72f2b64
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/DBUpdates.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.om.helpers;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Client side representation of DBUpdates.
+ */
+public class DBUpdates {
+
+ private List<byte[]> dataList = new ArrayList<>();
+
+ private long currentSequenceNumber = -1;
+
+ public DBUpdates() {
+ this.dataList = new ArrayList<>();
+ }
+
+ public DBUpdates(List<byte[]> data) {
+ this.dataList = new ArrayList<>(data);
+ }
+
+ public void addWriteBatch(byte[] data, long sequenceNumber) {
+ dataList.add(data);
+ if (currentSequenceNumber < sequenceNumber) {
+ currentSequenceNumber = sequenceNumber;
+ }
+ }
+
+ public List<byte[]> getData() {
+ return dataList;
+ }
+
+ public void setCurrentSequenceNumber(long sequenceNumber) {
+ this.currentSequenceNumber = sequenceNumber;
+ }
+
+ public long getCurrentSequenceNumber() {
+ return currentSequenceNumber;
+ }
+}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java
index 5dec9c6..f8c4d7a 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java
@@ -215,6 +215,8 @@
auditMap.put(OzoneConsts.STORAGE_TYPE,
(this.storageType != null) ? this.storageType.name() : null);
auditMap.put(OzoneConsts.CREATION_TIME, String.valueOf(this.creationTime));
+ auditMap.put(OzoneConsts.BUCKET_ENCRYPTION_KEY,
+ (bekInfo != null) ? bekInfo.getKeyName() : null);
return auditMap;
}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
index e825430..ee71fb2 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
@@ -408,6 +408,9 @@
}
public static OmKeyInfo getFromProtobuf(KeyInfo keyInfo) {
+ if (keyInfo == null) {
+ return null;
+ }
Builder builder = new Builder()
.setVolumeName(keyInfo.getVolumeName())
.setBucketName(keyInfo.getBucketName())
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFileStatus.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFileStatus.java
index 5f5f5b1..2ff69c3 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFileStatus.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFileStatus.java
@@ -18,105 +18,116 @@
package org.apache.hadoop.ozone.om.helpers;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.fs.protocolPB.PBHelper;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneFileStatusProto;
+import java.util.Objects;
-import java.io.IOException;
-import java.net.URI;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneFileStatusProto;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneFileStatusProto.Builder;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
/**
* File Status of the Ozone Key.
*/
-public class OzoneFileStatus extends FileStatus {
+public class OzoneFileStatus {
private static final long serialVersionUID = 1L;
- transient private OmKeyInfo keyInfo;
-
- public OzoneFileStatus(OmKeyInfo key, long blockSize, boolean isDirectory) {
- super(key.getDataSize(), isDirectory, key.getFactor().getNumber(),
- blockSize, key.getModificationTime(), getPath(key.getKeyName()));
- keyInfo = key;
- }
-
- public OzoneFileStatus(FileStatus status, OmKeyInfo key) throws IOException {
- super(status);
- keyInfo = key;
- }
-
- // Use this constructor only for directories
- public OzoneFileStatus(String keyName) {
- super(0, true, 0, 0, 0, getPath(keyName));
- }
-
- public OzoneFileStatusProto getProtobuf() throws IOException {
- OzoneFileStatusProto.Builder builder = OzoneFileStatusProto.newBuilder()
- .setStatus(PBHelper.convert(this));
- if (keyInfo != null) {
- builder.setKeyInfo(keyInfo.getProtobuf());
- }
- return builder.build();
- }
-
- public static OzoneFileStatus getFromProtobuf(OzoneFileStatusProto response)
- throws IOException {
- return new OzoneFileStatus(PBHelper.convert(response.getStatus()),
- OmKeyInfo.getFromProtobuf(response.getKeyInfo()));
- }
-
- public static Path getPath(String keyName) {
- return new Path(OZONE_URI_DELIMITER + keyName);
- }
-
- public FileStatus makeQualified(URI defaultUri, Path parent,
- String owner, String group) {
- // fully-qualify path
- setPath(parent.makeQualified(defaultUri, null));
- setGroup(group);
- setOwner(owner);
- if (isDirectory()) {
- setPermission(FsPermission.getDirDefault());
- } else {
- setPermission(FsPermission.getFileDefault());
- }
- return this; // API compatibility
- }
-
- /** Get the modification time of the file/directory.
- *
- * o3fs uses objects as "fake" directories, which are not updated to
- * reflect the accurate modification time. We choose to report the
- * current time because some parts of the ecosystem (e.g. the
- * HistoryServer) use modification time to ignore "old" directories.
- *
- * @return for files the modification time in milliseconds since January 1,
- * 1970 UTC or for directories the current time.
+ /**
+ * The key info object for files. Leave null for the root directory.
*/
- @Override
- public long getModificationTime(){
- if (isDirectory() && super.getModificationTime() == 0) {
- return System.currentTimeMillis();
- } else {
- return super.getModificationTime();
- }
+ private OmKeyInfo keyInfo;
+
+ private boolean isDirectory;
+
+ private long blockSize;
+
+ public OzoneFileStatus() {
+ isDirectory = true;
+ }
+
+ public OzoneFileStatus(OmKeyInfo keyInfo,
+ long blockSize, boolean isDirectory) {
+ this.keyInfo = keyInfo;
+ this.isDirectory = isDirectory;
+ this.blockSize = blockSize;
}
public OmKeyInfo getKeyInfo() {
return keyInfo;
}
+ public long getBlockSize() {
+ return blockSize;
+ }
+
+ public String getTrimmedName() {
+ String keyName = keyInfo.getKeyName();
+ if (keyName.endsWith(OZONE_URI_DELIMITER)) {
+ return keyName.substring(0, keyName.length() - 1);
+ } else {
+ return keyName;
+ }
+ }
+
+ public String getPath() {
+ if (keyInfo == null) {
+ return OZONE_URI_DELIMITER;
+ } else {
+ String path = OZONE_URI_DELIMITER + keyInfo.getKeyName();
+ if (path.endsWith(OZONE_URI_DELIMITER)) {
+ return path.substring(0, path.length() - 1);
+ } else {
+ return path;
+ }
+ }
+ }
+
+ public boolean isDirectory() {
+ if (keyInfo == null) {
+ return true;
+ }
+ return isDirectory;
+ }
+
+ public boolean isFile() {
+ return !isDirectory();
+ }
+
+ public OzoneFileStatusProto getProtobuf() {
+
+ Builder builder = OzoneFileStatusProto.newBuilder()
+ .setBlockSize(blockSize)
+ .setIsDirectory(isDirectory);
+ //key info can be null for the fake root entry.
+ if (keyInfo != null) {
+ builder.setKeyInfo(keyInfo.getProtobuf());
+ }
+ return builder.build();
+ }
+
+ public static OzoneFileStatus getFromProtobuf(OzoneFileStatusProto status) {
+ return new OzoneFileStatus(
+ OmKeyInfo.getFromProtobuf(status.getKeyInfo()),
+ status.getBlockSize(),
+ status.getIsDirectory());
+ }
+
@Override
public boolean equals(Object o) {
- return super.equals(o);
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof OzoneFileStatus)) {
+ return false;
+ }
+ OzoneFileStatus that = (OzoneFileStatus) o;
+ return isDirectory == that.isDirectory &&
+ blockSize == that.blockSize &&
+ getTrimmedName().equals(that.getTrimmedName());
}
@Override
public int hashCode() {
- return super.hashCode();
+ return Objects.hash(getTrimmedName());
}
-}
\ No newline at end of file
+}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ServiceInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ServiceInfo.java
index e569db8..77bd803 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ServiceInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ServiceInfo.java
@@ -19,33 +19,25 @@
package org.apache.hadoop.ozone.om.helpers;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
- .OMRoleInfo;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
- .ServicePort;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
-
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRoleInfo;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.google.common.base.Preconditions;
+
/**
* ServiceInfo holds the config details of Ozone services.
*/
public final class ServiceInfo {
- private static final ObjectReader READER =
- new ObjectMapper().readerFor(ServiceInfo.class);
- private static final ObjectWriter WRITER =
- new ObjectMapper().writerWithDefaultPrettyPrinter();
/**
* Type of node/service.
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneManagerLock.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneManagerLock.java
index 31f0924..4112941 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneManagerLock.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneManagerLock.java
@@ -26,7 +26,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.ozone.lock.LockManager;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_MANAGER_FAIR_LOCK_DEFAULT;
@@ -91,7 +91,7 @@
* Creates new OzoneManagerLock instance.
* @param conf Configuration object
*/
- public OzoneManagerLock(Configuration conf) {
+ public OzoneManagerLock(ConfigurationSource conf) {
boolean fair = conf.getBoolean(OZONE_MANAGER_FAIR_LOCK,
OZONE_MANAGER_FAIR_LOCK_DEFAULT);
manager = new LockManager<>(conf, fair);
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
index f6fa62d..5818ff1 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -18,12 +18,16 @@
package org.apache.hadoop.ozone.om.protocol;
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.List;
+
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
import org.apache.hadoop.ozone.OzoneAcl;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.ha.OMFailoverProxyProvider;
-
-import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.om.helpers.DBUpdates;
import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
@@ -44,17 +48,10 @@
import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-
import org.apache.hadoop.ozone.security.OzoneDelegationTokenSelector;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.apache.hadoop.security.KerberosInfo;
import org.apache.hadoop.security.token.TokenInfo;
-import org.apache.hadoop.hdds.utils.db.DBUpdatesWrapper;
-import org.apache.hadoop.hdds.utils.db.SequenceNumberNotFoundException;
/**
* Protocol to talk to OM.
@@ -82,9 +79,11 @@
* Changes the owner of a volume.
* @param volume - Name of the volume.
* @param owner - Name of the owner.
+ * @return true if operation succeeded, false if specified user is
+ * already the owner.
* @throws IOException
*/
- void setOwner(String volume, String owner) throws IOException;
+ boolean setOwner(String volume, String owner) throws IOException;
/**
* Changes the Quota on a volume.
@@ -121,7 +120,7 @@
void deleteVolume(String volume) throws IOException;
/**
- * Lists volume owned by a specific user.
+ * Lists volumes accessible by a specific user.
* @param userName - user name
* @param prefix - Filter prefix -- Return only entries that match this.
* @param prevKey - Previous key -- List starts from the next from the prevkey
@@ -297,55 +296,6 @@
*/
/**
- * Creates an S3 bucket inside Ozone manager and creates the mapping needed
- * to access via both S3 and Ozone.
- * @param userName - S3 user name.
- * @param s3BucketName - S3 bucket Name.
- * @throws IOException - On failure, throws an exception like Bucket exists.
- */
- void createS3Bucket(String userName, String s3BucketName) throws IOException;
-
- /**
- * Delets an S3 bucket inside Ozone manager and deletes the mapping.
- * @param s3BucketName - S3 bucket Name.
- * @throws IOException in case the bucket cannot be deleted.
- */
- void deleteS3Bucket(String s3BucketName) throws IOException;
-
- /**
- * Returns the Ozone Namespace for the S3Bucket. It will return the
- * OzoneVolume/OzoneBucketName.
- * @param s3BucketName - S3 Bucket Name.
- * @return String - The Ozone canonical name for this s3 bucket. This
- * string is useful for mounting an OzoneFS.
- * @throws IOException - Error is throw if the s3bucket does not exist.
- */
- String getOzoneBucketMapping(String s3BucketName) throws IOException;
-
- /**
- * Returns a list of buckets represented by {@link OmBucketInfo}
- * for the given user. Argument username is required, others
- * are optional.
- *
- * @param userName
- * user Name.
- * @param startBucketName
- * the start bucket name, only the buckets whose name is
- * after this value will be included in the result.
- * @param bucketPrefix
- * bucket name prefix, only the buckets whose name has
- * this prefix will be included in the result.
- * @param maxNumOfBuckets
- * the maximum number of buckets to return. It ensures
- * the size of the result will not exceed this limit.
- * @return a list of buckets.
- * @throws IOException
- */
- List<OmBucketInfo> listS3Buckets(String userName, String startBucketName,
- String bucketPrefix, int maxNumOfBuckets)
- throws IOException;
-
- /**
* Initiate multipart upload for the specified key.
* @param keyArgs
* @return MultipartInfo
@@ -522,9 +472,8 @@
* Get DB updates since a specific sequence number.
* @param dbUpdatesRequest request that encapsulates a sequence number.
* @return Wrapper containing the updates.
- * @throws SequenceNumberNotFoundException if db is unable to read the data.
*/
- DBUpdatesWrapper getDBUpdates(
+ DBUpdates getDBUpdates(
OzoneManagerProtocolProtos.DBUpdatesRequest dbUpdatesRequest)
throws IOException;
@@ -553,10 +502,12 @@
* @param bucketName - The bucket name.
* @param keyName - The key user want to recover.
* @param destinationBucket - The bucket user want to recover to.
- * @return The recoverTrash
+ * @return The result of recovering operation is success or not.
* @throws IOException
*/
- boolean recoverTrash(String volumeName, String bucketName, String keyName,
- String destinationBucket) throws IOException;
+ default boolean recoverTrash(String volumeName, String bucketName,
+ String keyName, String destinationBucket) throws IOException {
+ return false;
+ }
}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index 2e9dac3..b11b571 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -23,10 +23,9 @@
import java.util.List;
import java.util.stream.Collectors;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.tracing.TracingUtil;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.retry.RetryPolicy;
@@ -36,10 +35,11 @@
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.exceptions.OMLeaderNotReadyException;
import org.apache.hadoop.ozone.om.exceptions.OMNotLeaderException;
-import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.ha.OMFailoverProxyProvider;
+import org.apache.hadoop.ozone.om.helpers.DBUpdates;
import org.apache.hadoop.ozone.om.helpers.KeyValueUtil;
import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
@@ -61,25 +61,9 @@
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.AddAclResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetAclRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetAclResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListMultipartUploadsRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListMultipartUploadsResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneFileStatusProto;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LookupFileRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LookupFileResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateFileRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateFileResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListStatusRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListStatusResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetAclResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DBUpdatesRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DBUpdatesResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.AddAclRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateDirectoryRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetFileStatusResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetFileStatusRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.AddAclResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.AllocateBlockRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.AllocateBlockResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketArgs;
@@ -88,13 +72,22 @@
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CheckVolumeAccessRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CommitKeyRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateBucketRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateDirectoryRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateFileRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateFileResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateKeyRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateKeyResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateVolumeRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DBUpdatesRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DBUpdatesResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteBucketRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteVolumeRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetAclRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetAclResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetDelegationTokenResponseProto;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetFileStatusRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetFileStatusResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetS3SecretRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetS3SecretResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.InfoBucketRequest;
@@ -106,12 +99,16 @@
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListBucketsResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListKeysRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListKeysResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTrashResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RecoverTrashRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RecoverTrashResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListMultipartUploadsRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListMultipartUploadsResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListStatusRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListStatusResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTrashRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTrashResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListVolumeRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListVolumeResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LookupFileRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LookupFileResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LookupKeyRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LookupKeyResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.MultipartCommitUploadPartRequest;
@@ -126,19 +123,17 @@
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneFileStatusProto;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RecoverTrashRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RecoverTrashResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RemoveAclRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RemoveAclResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenameKeyRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenewDelegationTokenResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3BucketInfoRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3BucketInfoResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3CreateBucketRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3DeleteBucketRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3ListBucketsRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3ListBucketsResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetAclRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetAclResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetBucketPropertyRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetVolumePropertyRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
@@ -151,24 +146,23 @@
import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto;
import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProto;
+import org.apache.hadoop.security.token.SecretManager;
import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.hdds.utils.db.DBUpdatesWrapper;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.protobuf.ByteString;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.hadoop.io.retry.RetryPolicy.RetryAction.FAILOVER_AND_RETRY;
+import static org.apache.hadoop.io.retry.RetryPolicy.RetryAction.RetryDecision;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.TOKEN_ERROR_OTHER;
import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.ACCESS_DENIED;
import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.DIRECTORY_ALREADY_EXISTS;
import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* The client side implementation of OzoneManagerProtocol.
@@ -202,7 +196,7 @@
* one {@link OzoneManagerProtocolPB} proxy pointing to each OM node in the
* cluster.
*/
- public OzoneManagerProtocolClientSideTranslatorPB(OzoneConfiguration conf,
+ public OzoneManagerProtocolClientSideTranslatorPB(ConfigurationSource conf,
String clientId, String omServiceId, UserGroupInformation ugi)
throws IOException {
this.omFailoverProxyProvider = new OMFailoverProxyProvider(conf, ugi,
@@ -211,15 +205,8 @@
int maxFailovers = conf.getInt(
OzoneConfigKeys.OZONE_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY,
OzoneConfigKeys.OZONE_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT);
- int sleepBase = conf.getInt(
- OzoneConfigKeys.OZONE_CLIENT_FAILOVER_SLEEP_BASE_MILLIS_KEY,
- OzoneConfigKeys.OZONE_CLIENT_FAILOVER_SLEEP_BASE_MILLIS_DEFAULT);
- int sleepMax = conf.getInt(
- OzoneConfigKeys.OZONE_CLIENT_FAILOVER_SLEEP_MAX_MILLIS_KEY,
- OzoneConfigKeys.OZONE_CLIENT_FAILOVER_SLEEP_MAX_MILLIS_DEFAULT);
- this.rpcProxy = createRetryProxy(omFailoverProxyProvider, maxFailovers,
- sleepBase, sleepMax);
+ this.rpcProxy = createRetryProxy(omFailoverProxyProvider, maxFailovers);
this.clientID = clientId;
}
@@ -229,8 +216,7 @@
* exception or if the current proxy is not the leader OM.
*/
private OzoneManagerProtocolPB createRetryProxy(
- OMFailoverProxyProvider failoverProxyProvider,
- int maxFailovers, int delayMillis, int maxDelayBase) {
+ OMFailoverProxyProvider failoverProxyProvider, int maxFailovers) {
// Client attempts contacting each OM ipc.client.connect.max.retries
// (default = 10) times before failing over to the next OM, if
@@ -250,12 +236,17 @@
getNotLeaderException(exception);
if (notLeaderException != null &&
notLeaderException.getSuggestedLeaderNodeId() != null) {
- // We need to failover manually to the suggested Leader OM Node.
+ FAILOVER_PROXY_PROVIDER_LOG.info("RetryProxy: {}",
+ notLeaderException.getMessage());
+
+ // TODO: NotLeaderException should include the host
+ // address of the suggested leader along with the nodeID.
+ // Failing over just based on nodeID is not very robust.
+
// OMFailoverProxyProvider#performFailover() is a dummy call and
- // does not perform any failover.
- omFailoverProxyProvider.performFailoverIfRequired(
- notLeaderException.getSuggestedLeaderNodeId());
- return getRetryAction(FAILOVER_AND_RETRY, failovers);
+ // does not perform any failover. Failover manually to the next OM.
+ omFailoverProxyProvider.performFailoverToNextProxy();
+ return getRetryAction(RetryDecision.FAILOVER_AND_RETRY, failovers);
}
OMLeaderNotReadyException leaderNotReadyException =
@@ -263,9 +254,17 @@
// As in this case, current OM node is leader, but it is not ready.
// OMFailoverProxyProvider#performFailover() is a dummy call and
// does not perform any failover.
- // So Just retry with same ON node.
+ // So Just retry with same OM node.
if (leaderNotReadyException != null) {
- return getRetryAction(FAILOVER_AND_RETRY, failovers);
+ FAILOVER_PROXY_PROVIDER_LOG.info("RetryProxy: {}",
+ leaderNotReadyException.getMessage());
+ // HDDS-3465. OM index will not change, but LastOmID will be
+ // updated to currentOMId, so that wiatTime calculation will
+ // know lastOmID and currentID are same and need to increment
+ // wait time in between.
+ omFailoverProxyProvider.performFailoverIfRequired(
+ omFailoverProxyProvider.getCurrentProxyOMNodeId());
+ return getRetryAction(RetryDecision.FAILOVER_AND_RETRY, failovers);
}
}
@@ -273,14 +272,22 @@
// NotLeaderException fail over manually to the next OM Node proxy.
// OMFailoverProxyProvider#performFailover() is a dummy call and
// does not perform any failover.
+ String exceptionMsg;
+ if (exception.getCause() != null) {
+ exceptionMsg = exception.getCause().getMessage();
+ } else {
+ exceptionMsg = exception.getMessage();
+ }
+ FAILOVER_PROXY_PROVIDER_LOG.info("RetryProxy: {}", exceptionMsg);
omFailoverProxyProvider.performFailoverToNextProxy();
- return getRetryAction(FAILOVER_AND_RETRY, failovers);
+ return getRetryAction(RetryDecision.FAILOVER_AND_RETRY, failovers);
}
- private RetryAction getRetryAction(RetryAction fallbackAction,
+ private RetryAction getRetryAction(RetryDecision fallbackAction,
int failovers) {
- if (failovers <= maxFailovers) {
- return fallbackAction;
+ if (failovers < maxFailovers) {
+ return new RetryAction(fallbackAction,
+ omFailoverProxyProvider.getWaitTime());
} else {
FAILOVER_PROXY_PROVIDER_LOG.error("Failed to connect to OMs: {}. " +
"Attempted {} failovers.",
@@ -296,13 +303,18 @@
}
/**
- * Unwrap exception to check if it is a {@link AccessControlException}.
+ * Unwrap exception to check if it is some kind of access control problem
+ * ({@link AccessControlException} or {@link SecretManager.InvalidToken}).
*/
private boolean isAccessControlException(Exception ex) {
if (ex instanceof ServiceException) {
Throwable t = ex.getCause();
+ if (t instanceof RemoteException) {
+ t = ((RemoteException) t).unwrapRemoteException();
+ }
while (t != null) {
- if (t instanceof AccessControlException) {
+ if (t instanceof AccessControlException ||
+ t instanceof SecretManager.InvalidToken) {
return true;
}
t = t.getCause();
@@ -446,14 +458,10 @@
}
/**
- * Changes the owner of a volume.
- *
- * @param volume - Name of the volume.
- * @param owner - Name of the owner.
- * @throws IOException
+ * {@inheritDoc}
*/
@Override
- public void setOwner(String volume, String owner) throws IOException {
+ public boolean setOwner(String volume, String owner) throws IOException {
SetVolumePropertyRequest.Builder req =
SetVolumePropertyRequest.newBuilder();
req.setVolumeName(volume).setOwnerName(owner);
@@ -463,7 +471,10 @@
.build();
OMResponse omResponse = submitRequest(omRequest);
- handleError(omResponse);
+ OzoneManagerProtocolProtos.SetVolumePropertyResponse response =
+ handleError(omResponse).getSetVolumePropertyResponse();
+
+ return response.getResponse();
}
/**
@@ -562,7 +573,7 @@
}
/**
- * Lists volume owned by a specific user.
+ * Lists volumes accessible by a specific user.
*
* @param userName - user name
* @param prefix - Filter prefix -- Return only entries that match this.
@@ -984,83 +995,6 @@
}
@Override
- public void createS3Bucket(String userName, String s3BucketName)
- throws IOException {
- S3CreateBucketRequest req = S3CreateBucketRequest.newBuilder()
- .setUserName(userName)
- .setS3Bucketname(s3BucketName)
- .build();
-
- OMRequest omRequest = createOMRequest(Type.CreateS3Bucket)
- .setCreateS3BucketRequest(req)
- .build();
-
- handleError(submitRequest(omRequest));
-
- }
-
- @Override
- public void deleteS3Bucket(String s3BucketName) throws IOException {
- S3DeleteBucketRequest request = S3DeleteBucketRequest.newBuilder()
- .setS3BucketName(s3BucketName)
- .build();
-
- OMRequest omRequest = createOMRequest(Type.DeleteS3Bucket)
- .setDeleteS3BucketRequest(request)
- .build();
-
- handleError(submitRequest(omRequest));
-
- }
-
- @Override
- public String getOzoneBucketMapping(String s3BucketName)
- throws IOException {
- S3BucketInfoRequest request = S3BucketInfoRequest.newBuilder()
- .setS3BucketName(s3BucketName)
- .build();
-
- OMRequest omRequest = createOMRequest(Type.InfoS3Bucket)
- .setInfoS3BucketRequest(request)
- .build();
-
- S3BucketInfoResponse resp = handleError(submitRequest(omRequest))
- .getInfoS3BucketResponse();
- return resp.getOzoneMapping();
- }
-
- @Override
- public List<OmBucketInfo> listS3Buckets(String userName, String startKey,
- String prefix, int count)
- throws IOException {
- List<OmBucketInfo> buckets = new ArrayList<>();
- S3ListBucketsRequest.Builder reqBuilder = S3ListBucketsRequest.newBuilder();
- reqBuilder.setUserName(userName);
- reqBuilder.setCount(count);
- if (startKey != null) {
- reqBuilder.setStartKey(startKey);
- }
- if (prefix != null) {
- reqBuilder.setPrefix(prefix);
- }
- S3ListBucketsRequest request = reqBuilder.build();
-
- OMRequest omRequest = createOMRequest(Type.ListS3Buckets)
- .setListS3BucketsRequest(request)
- .build();
-
- S3ListBucketsResponse resp = handleError(submitRequest(omRequest))
- .getListS3BucketsResponse();
-
- buckets.addAll(
- resp.getBucketInfoList().stream()
- .map(OmBucketInfo::getFromProtobuf)
- .collect(Collectors.toList()));
- return buckets;
-
- }
-
- @Override
public S3SecretValue getS3Secret(String kerberosID) throws IOException {
GetS3SecretRequest request = GetS3SecretRequest.newBuilder()
.setKerberosID(kerberosID)
@@ -1572,7 +1506,7 @@
}
@Override
- public DBUpdatesWrapper getDBUpdates(DBUpdatesRequest dbUpdatesRequest)
+ public DBUpdates getDBUpdates(DBUpdatesRequest dbUpdatesRequest)
throws IOException {
OMRequest omRequest = createOMRequest(Type.DBUpdates)
.setDbUpdatesRequest(dbUpdatesRequest)
@@ -1581,7 +1515,7 @@
DBUpdatesResponse dbUpdatesResponse =
handleError(submitRequest(omRequest)).getDbUpdatesResponse();
- DBUpdatesWrapper dbUpdatesWrapper = new DBUpdatesWrapper();
+ DBUpdates dbUpdatesWrapper = new DBUpdates();
for (ByteString byteString : dbUpdatesResponse.getDataList()) {
dbUpdatesWrapper.addWriteBatch(byteString.toByteArray(), 0L);
}
@@ -1705,15 +1639,14 @@
"The destination bucket name cannot be null or empty. " +
"Please enter a valid destination bucket name.");
- RecoverTrashRequest recoverRequest = RecoverTrashRequest.newBuilder()
+ RecoverTrashRequest.Builder req = RecoverTrashRequest.newBuilder()
.setVolumeName(volumeName)
.setBucketName(bucketName)
.setKeyName(keyName)
- .setDestinationBucket(destinationBucket)
- .build();
+ .setDestinationBucket(destinationBucket);
OMRequest omRequest = createOMRequest(Type.RecoverTrash)
- .setRecoverTrashRequest(recoverRequest)
+ .setRecoverTrashRequest(req)
.build();
RecoverTrashResponse recoverResponse =
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java
index 13124c3..290dd1d 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java
@@ -47,6 +47,7 @@
private String awsAccessId;
private String signature;
private String strToSign;
+ private String omServiceId;
/**
* Create an empty delegation token identifier.
@@ -102,7 +103,11 @@
.setStrToSign(getStrToSign());
} else {
builder.setOmCertSerialId(getOmCertSerialId());
+ if (getOmServiceId() != null) {
+ builder.setOmServiceId(getOmServiceId());
+ }
}
+
OMTokenProto token = builder.build();
out.write(token.toByteArray());
}
@@ -133,6 +138,10 @@
setSignature(token.getSignature());
setStrToSign(token.getStrToSign());
}
+
+ if (token.hasOmServiceId()) {
+ setOmServiceId(token.getOmServiceId());
+ }
}
/**
@@ -160,6 +169,7 @@
identifier.setMasterKeyId(token.getMasterKeyId());
}
identifier.setOmCertSerialId(token.getOmCertSerialId());
+ identifier.setOmServiceId(token.getOmServiceId());
return identifier;
}
@@ -210,6 +220,7 @@
.append(getRenewer(), that.getRenewer())
.append(getKind(), that.getKind())
.append(getSequenceNumber(), that.getSequenceNumber())
+ .append(getOmServiceId(), that.getOmServiceId())
.build();
}
@@ -264,6 +275,14 @@
this.omCertSerialId = omCertSerialId;
}
+ public String getOmServiceId() {
+ return omServiceId;
+ }
+
+ public void setOmServiceId(String omServiceId) {
+ this.omServiceId = omServiceId;
+ }
+
public Type getTokenType() {
return tokenType;
}
@@ -309,7 +328,8 @@
.append(", masterKeyId=").append(getMasterKeyId())
.append(", strToSign=").append(getStrToSign())
.append(", signature=").append(getSignature())
- .append(", awsAccessKeyId=").append(getAwsAccessId());
+ .append(", awsAccessKeyId=").append(getAwsAccessId())
+ .append(", omServiceId=").append(getOmServiceId());
return buffer.toString();
}
}
\ No newline at end of file
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java
index ae295e3..1cdea8b 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java
@@ -29,7 +29,7 @@
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.ozone.OzoneConsts;
@@ -148,7 +148,8 @@
* Return the TimeDuration configured for the given key. If not configured,
* return the default value.
*/
- public static TimeDuration getTimeDuration(Configuration conf, String key,
+ public static TimeDuration getTimeDuration(ConfigurationSource conf,
+ String key,
TimeDuration defaultValue) {
TimeUnit defaultTimeUnit = defaultValue.getUnit();
long timeDurationInDefaultUnit = conf.getTimeDuration(key,
@@ -159,7 +160,7 @@
/**
* Return the time configured for the given key in milliseconds.
*/
- public static long getTimeDurationInMS(Configuration conf, String key,
+ public static long getTimeDurationInMS(ConfigurationSource conf, String key,
TimeDuration defaultValue) {
return getTimeDuration(conf, key, defaultValue)
.toLong(TimeUnit.MILLISECONDS);
diff --git a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
index 5237136..569be74 100644
--- a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
+++ b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
@@ -37,7 +37,6 @@
import "hdds.proto";
import "Security.proto";
-import "FSProtos.proto";
enum Type {
CreateVolume = 11;
@@ -61,10 +60,6 @@
CommitKey = 36;
AllocateBlock = 37;
- CreateS3Bucket = 41;
- DeleteS3Bucket = 42;
- InfoS3Bucket = 43;
- ListS3Buckets = 44;
InitiateMultiPartUpload = 45;
CommitMultiPartUpload = 46;
CompleteMultiPartUpload = 47;
@@ -130,10 +125,6 @@
optional CommitKeyRequest commitKeyRequest = 36;
optional AllocateBlockRequest allocateBlockRequest = 37;
- optional S3CreateBucketRequest createS3BucketRequest = 41;
- optional S3DeleteBucketRequest deleteS3BucketRequest = 42;
- optional S3BucketInfoRequest infoS3BucketRequest = 43;
- optional S3ListBucketsRequest listS3BucketsRequest = 44;
optional MultipartInfoInitiateRequest initiateMultiPartUploadRequest = 45;
optional MultipartCommitUploadPartRequest commitMultiPartUploadRequest = 46;
optional MultipartUploadCompleteRequest completeMultiPartUploadRequest = 47;
@@ -205,10 +196,6 @@
optional CommitKeyResponse commitKeyResponse = 36;
optional AllocateBlockResponse allocateBlockResponse = 37;
- optional S3CreateBucketResponse createS3BucketResponse = 41;
- optional S3DeleteBucketResponse deleteS3BucketResponse = 42;
- optional S3BucketInfoResponse infoS3BucketResponse = 43;
- optional S3ListBucketsResponse listS3BucketsResponse = 44;
optional MultipartInfoInitiateResponse initiateMultiPartUploadResponse = 45;
optional MultipartCommitUploadPartResponse commitMultiPartUploadResponse = 46;
optional MultipartUploadCompleteResponse completeMultiPartUploadResponse = 47;
@@ -263,8 +250,6 @@
METADATA_ERROR = 19;
OM_NOT_INITIALIZED = 20;
SCM_VERSION_MISMATCH_ERROR = 21;
- S3_BUCKET_NOT_FOUND = 22;
- S3_BUCKET_ALREADY_EXISTS = 23;
INITIATE_MULTIPART_UPLOAD_ERROR = 24;
MULTIPART_UPLOAD_PARTFILE_ERROR = 25;
@@ -298,8 +283,6 @@
TIMEOUT = 49;
PREFIX_NOT_FOUND=50;
- S3_BUCKET_INVALID_LENGTH = 51; // s3 bucket invalid length.
-
RATIS_ERROR = 52;
INVALID_PATH_IN_ACL_REQUEST = 53; // Invalid path name in acl request.
@@ -315,6 +298,8 @@
CANNOT_CREATE_DIRECTORY_AT_ROOT = 59;
DIRECTORY_ALREADY_EXISTS = 60;
+ INVALID_VOLUME_NAME = 61;
+
// When transactions are replayed
REPLAY = 100;
}
@@ -421,7 +406,7 @@
}
message SetVolumePropertyResponse {
-
+ optional bool response = 1;
}
/**
@@ -707,6 +692,9 @@
// request type.
optional uint64 modificationTime = 13;
optional bool sortDatanodes = 14;
+
+ // This will be set by leader OM in HA and update the original request.
+ optional FileEncryptionInfoProto fileEncryptionInfo = 15;
}
message KeyLocation {
@@ -754,8 +742,10 @@
}
message OzoneFileStatusProto {
- required hadoop.fs.FileStatusProto status = 1;
optional KeyInfo keyInfo = 2;
+ optional uint64 blockSize = 3;
+ optional bool isDirectory = 4;
+
}
message GetFileStatusRequest {
@@ -893,6 +883,7 @@
optional string accessKeyId = 12;
optional string signature = 13;
optional string strToSign = 14;
+ optional string omServiceId = 15;
}
message SecretKeyProto {
@@ -981,53 +972,6 @@
optional OMRoleInfo omRole = 4;
}
-message S3CreateBucketRequest {
- required string userName = 1;
- required string s3bucketname = 2;
- // This will be set during OM HA by one of the OM node. In future if more
- // data fields are required to create volume/bucket we can add them to
- // this. This is the reason for creating a new message type for this.
- // S3CreateBucket means create volume from userName and create bucket
- // with s3BucketName.
- optional S3CreateVolumeInfo s3CreateVolumeInfo = 3;
-}
-
-message S3CreateVolumeInfo {
- // Creation time set in preExecute on one of the OM node.
- required uint64 creationTime = 1;
-}
-
-message S3CreateBucketResponse {
-
-}
-
-message S3DeleteBucketRequest {
- required string s3bucketName = 1;
-}
-
-message S3DeleteBucketResponse {
-
-}
-
-message S3BucketInfoRequest {
- required string s3bucketName = 1;
-}
-message S3BucketInfoResponse {
-
- optional string ozoneMapping = 2;
-}
-
-message S3ListBucketsRequest {
- required string userName = 1;
- optional string startKey = 2;
- optional string prefix = 3;
- optional int32 count = 4;
-}
-
-message S3ListBucketsResponse {
- repeated BucketInfo bucketInfo = 2;
-}
-
message MultipartInfoInitiateRequest {
required KeyArgs keyArgs = 1;
diff --git a/hadoop-ozone/common/src/main/proto/proto.lock b/hadoop-ozone/common/src/main/proto/proto.lock
new file mode 100644
index 0000000..0de831c
--- /dev/null
+++ b/hadoop-ozone/common/src/main/proto/proto.lock
@@ -0,0 +1,3240 @@
+{
+ "definitions": [
+ {
+ "protopath": "OzoneManagerProtocol.proto",
+ "def": {
+ "enums": [
+ {
+ "name": "Type",
+ "enum_fields": [
+ {
+ "name": "CreateVolume",
+ "integer": 11
+ },
+ {
+ "name": "SetVolumeProperty",
+ "integer": 12
+ },
+ {
+ "name": "CheckVolumeAccess",
+ "integer": 13
+ },
+ {
+ "name": "InfoVolume",
+ "integer": 14
+ },
+ {
+ "name": "DeleteVolume",
+ "integer": 15
+ },
+ {
+ "name": "ListVolume",
+ "integer": 16
+ },
+ {
+ "name": "CreateBucket",
+ "integer": 21
+ },
+ {
+ "name": "InfoBucket",
+ "integer": 22
+ },
+ {
+ "name": "SetBucketProperty",
+ "integer": 23
+ },
+ {
+ "name": "DeleteBucket",
+ "integer": 24
+ },
+ {
+ "name": "ListBuckets",
+ "integer": 25
+ },
+ {
+ "name": "CreateKey",
+ "integer": 31
+ },
+ {
+ "name": "LookupKey",
+ "integer": 32
+ },
+ {
+ "name": "RenameKey",
+ "integer": 33
+ },
+ {
+ "name": "DeleteKey",
+ "integer": 34
+ },
+ {
+ "name": "ListKeys",
+ "integer": 35
+ },
+ {
+ "name": "CommitKey",
+ "integer": 36
+ },
+ {
+ "name": "AllocateBlock",
+ "integer": 37
+ },
+ {
+ "name": "InitiateMultiPartUpload",
+ "integer": 45
+ },
+ {
+ "name": "CommitMultiPartUpload",
+ "integer": 46
+ },
+ {
+ "name": "CompleteMultiPartUpload",
+ "integer": 47
+ },
+ {
+ "name": "AbortMultiPartUpload",
+ "integer": 48
+ },
+ {
+ "name": "GetS3Secret",
+ "integer": 49
+ },
+ {
+ "name": "ListMultiPartUploadParts",
+ "integer": 50
+ },
+ {
+ "name": "ServiceList",
+ "integer": 51
+ },
+ {
+ "name": "DBUpdates",
+ "integer": 53
+ },
+ {
+ "name": "GetDelegationToken",
+ "integer": 61
+ },
+ {
+ "name": "RenewDelegationToken",
+ "integer": 62
+ },
+ {
+ "name": "CancelDelegationToken",
+ "integer": 63
+ },
+ {
+ "name": "GetFileStatus",
+ "integer": 70
+ },
+ {
+ "name": "CreateDirectory",
+ "integer": 71
+ },
+ {
+ "name": "CreateFile",
+ "integer": 72
+ },
+ {
+ "name": "LookupFile",
+ "integer": 73
+ },
+ {
+ "name": "ListStatus",
+ "integer": 74
+ },
+ {
+ "name": "AddAcl",
+ "integer": 75
+ },
+ {
+ "name": "RemoveAcl",
+ "integer": 76
+ },
+ {
+ "name": "SetAcl",
+ "integer": 77
+ },
+ {
+ "name": "GetAcl",
+ "integer": 78
+ },
+ {
+ "name": "PurgeKeys",
+ "integer": 81
+ },
+ {
+ "name": "ListMultipartUploads",
+ "integer": 82
+ },
+ {
+ "name": "ListTrash",
+ "integer": 91
+ },
+ {
+ "name": "RecoverTrash",
+ "integer": 92
+ }
+ ]
+ },
+ {
+ "name": "Status",
+ "enum_fields": [
+ {
+ "name": "OK",
+ "integer": 1
+ },
+ {
+ "name": "VOLUME_NOT_UNIQUE",
+ "integer": 2
+ },
+ {
+ "name": "VOLUME_NOT_FOUND",
+ "integer": 3
+ },
+ {
+ "name": "VOLUME_NOT_EMPTY",
+ "integer": 4
+ },
+ {
+ "name": "VOLUME_ALREADY_EXISTS",
+ "integer": 5
+ },
+ {
+ "name": "USER_NOT_FOUND",
+ "integer": 6
+ },
+ {
+ "name": "USER_TOO_MANY_VOLUMES",
+ "integer": 7
+ },
+ {
+ "name": "BUCKET_NOT_FOUND",
+ "integer": 8
+ },
+ {
+ "name": "BUCKET_NOT_EMPTY",
+ "integer": 9
+ },
+ {
+ "name": "BUCKET_ALREADY_EXISTS",
+ "integer": 10
+ },
+ {
+ "name": "KEY_ALREADY_EXISTS",
+ "integer": 11
+ },
+ {
+ "name": "KEY_NOT_FOUND",
+ "integer": 12
+ },
+ {
+ "name": "INVALID_KEY_NAME",
+ "integer": 13
+ },
+ {
+ "name": "ACCESS_DENIED",
+ "integer": 14
+ },
+ {
+ "name": "INTERNAL_ERROR",
+ "integer": 15
+ },
+ {
+ "name": "KEY_ALLOCATION_ERROR",
+ "integer": 16
+ },
+ {
+ "name": "KEY_DELETION_ERROR",
+ "integer": 17
+ },
+ {
+ "name": "KEY_RENAME_ERROR",
+ "integer": 18
+ },
+ {
+ "name": "METADATA_ERROR",
+ "integer": 19
+ },
+ {
+ "name": "OM_NOT_INITIALIZED",
+ "integer": 20
+ },
+ {
+ "name": "SCM_VERSION_MISMATCH_ERROR",
+ "integer": 21
+ },
+ {
+ "name": "INITIATE_MULTIPART_UPLOAD_ERROR",
+ "integer": 24
+ },
+ {
+ "name": "MULTIPART_UPLOAD_PARTFILE_ERROR",
+ "integer": 25
+ },
+ {
+ "name": "NO_SUCH_MULTIPART_UPLOAD_ERROR",
+ "integer": 26
+ },
+ {
+ "name": "MISMATCH_MULTIPART_LIST",
+ "integer": 27
+ },
+ {
+ "name": "MISSING_UPLOAD_PARTS",
+ "integer": 28
+ },
+ {
+ "name": "COMPLETE_MULTIPART_UPLOAD_ERROR",
+ "integer": 29
+ },
+ {
+ "name": "ENTITY_TOO_SMALL",
+ "integer": 30
+ },
+ {
+ "name": "ABORT_MULTIPART_UPLOAD_FAILED",
+ "integer": 31
+ },
+ {
+ "name": "S3_SECRET_NOT_FOUND",
+ "integer": 32
+ },
+ {
+ "name": "INVALID_AUTH_METHOD",
+ "integer": 33
+ },
+ {
+ "name": "INVALID_TOKEN",
+ "integer": 34
+ },
+ {
+ "name": "TOKEN_EXPIRED",
+ "integer": 35
+ },
+ {
+ "name": "TOKEN_ERROR_OTHER",
+ "integer": 36
+ },
+ {
+ "name": "LIST_MULTIPART_UPLOAD_PARTS_FAILED",
+ "integer": 37
+ },
+ {
+ "name": "SCM_IN_SAFE_MODE",
+ "integer": 38
+ },
+ {
+ "name": "INVALID_REQUEST",
+ "integer": 39
+ },
+ {
+ "name": "BUCKET_ENCRYPTION_KEY_NOT_FOUND",
+ "integer": 40
+ },
+ {
+ "name": "UNKNOWN_CIPHER_SUITE",
+ "integer": 41
+ },
+ {
+ "name": "INVALID_KMS_PROVIDER",
+ "integer": 42
+ },
+ {
+ "name": "TOKEN_CREATION_ERROR",
+ "integer": 43
+ },
+ {
+ "name": "FILE_NOT_FOUND",
+ "integer": 44
+ },
+ {
+ "name": "DIRECTORY_NOT_FOUND",
+ "integer": 45
+ },
+ {
+ "name": "FILE_ALREADY_EXISTS",
+ "integer": 46
+ },
+ {
+ "name": "NOT_A_FILE",
+ "integer": 47
+ },
+ {
+ "name": "PERMISSION_DENIED",
+ "integer": 48
+ },
+ {
+ "name": "TIMEOUT",
+ "integer": 49
+ },
+ {
+ "name": "PREFIX_NOT_FOUND",
+ "integer": 50
+ },
+ {
+ "name": "RATIS_ERROR",
+ "integer": 52
+ },
+ {
+ "name": "INVALID_PATH_IN_ACL_REQUEST",
+ "integer": 53
+ },
+ {
+ "name": "USER_MISMATCH",
+ "integer": 54
+ },
+ {
+ "name": "INVALID_PART",
+ "integer": 55
+ },
+ {
+ "name": "INVALID_PART_ORDER",
+ "integer": 56
+ },
+ {
+ "name": "SCM_GET_PIPELINE_EXCEPTION",
+ "integer": 57
+ },
+ {
+ "name": "INVALID_BUCKET_NAME",
+ "integer": 58
+ },
+ {
+ "name": "CANNOT_CREATE_DIRECTORY_AT_ROOT",
+ "integer": 59
+ },
+ {
+ "name": "DIRECTORY_ALREADY_EXISTS",
+ "integer": 60
+ },
+ {
+ "name": "INVALID_VOLUME_NAME",
+ "integer": 61
+ },
+ {
+ "name": "REPLAY",
+ "integer": 100
+ }
+ ]
+ },
+ {
+ "name": "ListVolumeRequest.Scope",
+ "enum_fields": [
+ {
+ "name": "USER_VOLUMES",
+ "integer": 1
+ },
+ {
+ "name": "VOLUMES_BY_USER",
+ "integer": 2
+ },
+ {
+ "name": "VOLUMES_BY_CLUSTER",
+ "integer": 3
+ }
+ ]
+ },
+ {
+ "name": "StorageTypeProto",
+ "enum_fields": [
+ {
+ "name": "DISK",
+ "integer": 1
+ },
+ {
+ "name": "SSD",
+ "integer": 2
+ },
+ {
+ "name": "ARCHIVE",
+ "integer": 3
+ },
+ {
+ "name": "RAM_DISK",
+ "integer": 4
+ }
+ ]
+ },
+ {
+ "name": "CipherSuiteProto",
+ "enum_fields": [
+ {
+ "name": "UNKNOWN",
+ "integer": 1
+ },
+ {
+ "name": "AES_CTR_NOPADDING",
+ "integer": 2
+ }
+ ]
+ },
+ {
+ "name": "CryptoProtocolVersionProto",
+ "enum_fields": [
+ {
+ "name": "UNKNOWN_PROTOCOL_VERSION",
+ "integer": 1
+ },
+ {
+ "name": "ENCRYPTION_ZONES",
+ "integer": 2
+ }
+ ]
+ },
+ {
+ "name": "OzoneObj.ObjectType",
+ "enum_fields": [
+ {
+ "name": "VOLUME",
+ "integer": 1
+ },
+ {
+ "name": "BUCKET",
+ "integer": 2
+ },
+ {
+ "name": "KEY",
+ "integer": 3
+ },
+ {
+ "name": "PREFIX",
+ "integer": 4
+ }
+ ]
+ },
+ {
+ "name": "OzoneObj.StoreType",
+ "enum_fields": [
+ {
+ "name": "OZONE",
+ "integer": 1
+ },
+ {
+ "name": "S3",
+ "integer": 2
+ }
+ ]
+ },
+ {
+ "name": "OzoneAclInfo.OzoneAclType",
+ "enum_fields": [
+ {
+ "name": "USER",
+ "integer": 1
+ },
+ {
+ "name": "GROUP",
+ "integer": 2
+ },
+ {
+ "name": "WORLD",
+ "integer": 3
+ },
+ {
+ "name": "ANONYMOUS",
+ "integer": 4
+ },
+ {
+ "name": "CLIENT_IP",
+ "integer": 5
+ }
+ ]
+ },
+ {
+ "name": "OzoneAclInfo.OzoneAclScope",
+ "enum_fields": [
+ {
+ "name": "ACCESS"
+ },
+ {
+ "name": "DEFAULT",
+ "integer": 1
+ }
+ ]
+ },
+ {
+ "name": "OMTokenProto.Type",
+ "enum_fields": [
+ {
+ "name": "DELEGATION_TOKEN",
+ "integer": 1
+ },
+ {
+ "name": "S3AUTHINFO",
+ "integer": 2
+ }
+ ]
+ },
+ {
+ "name": "ServicePort.Type",
+ "enum_fields": [
+ {
+ "name": "RPC",
+ "integer": 1
+ },
+ {
+ "name": "HTTP",
+ "integer": 2
+ },
+ {
+ "name": "HTTPS",
+ "integer": 3
+ },
+ {
+ "name": "RATIS",
+ "integer": 4
+ }
+ ]
+ }
+ ],
+ "messages": [
+ {
+ "name": "OMRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "cmdType",
+ "type": "Type"
+ },
+ {
+ "id": 2,
+ "name": "traceID",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "clientId",
+ "type": "string"
+ },
+ {
+ "id": 4,
+ "name": "userInfo",
+ "type": "UserInfo"
+ },
+ {
+ "id": 11,
+ "name": "createVolumeRequest",
+ "type": "CreateVolumeRequest"
+ },
+ {
+ "id": 12,
+ "name": "setVolumePropertyRequest",
+ "type": "SetVolumePropertyRequest"
+ },
+ {
+ "id": 13,
+ "name": "checkVolumeAccessRequest",
+ "type": "CheckVolumeAccessRequest"
+ },
+ {
+ "id": 14,
+ "name": "infoVolumeRequest",
+ "type": "InfoVolumeRequest"
+ },
+ {
+ "id": 15,
+ "name": "deleteVolumeRequest",
+ "type": "DeleteVolumeRequest"
+ },
+ {
+ "id": 16,
+ "name": "listVolumeRequest",
+ "type": "ListVolumeRequest"
+ },
+ {
+ "id": 21,
+ "name": "createBucketRequest",
+ "type": "CreateBucketRequest"
+ },
+ {
+ "id": 22,
+ "name": "infoBucketRequest",
+ "type": "InfoBucketRequest"
+ },
+ {
+ "id": 23,
+ "name": "setBucketPropertyRequest",
+ "type": "SetBucketPropertyRequest"
+ },
+ {
+ "id": 24,
+ "name": "deleteBucketRequest",
+ "type": "DeleteBucketRequest"
+ },
+ {
+ "id": 25,
+ "name": "listBucketsRequest",
+ "type": "ListBucketsRequest"
+ },
+ {
+ "id": 31,
+ "name": "createKeyRequest",
+ "type": "CreateKeyRequest"
+ },
+ {
+ "id": 32,
+ "name": "lookupKeyRequest",
+ "type": "LookupKeyRequest"
+ },
+ {
+ "id": 33,
+ "name": "renameKeyRequest",
+ "type": "RenameKeyRequest"
+ },
+ {
+ "id": 34,
+ "name": "deleteKeyRequest",
+ "type": "DeleteKeyRequest"
+ },
+ {
+ "id": 35,
+ "name": "listKeysRequest",
+ "type": "ListKeysRequest"
+ },
+ {
+ "id": 36,
+ "name": "commitKeyRequest",
+ "type": "CommitKeyRequest"
+ },
+ {
+ "id": 37,
+ "name": "allocateBlockRequest",
+ "type": "AllocateBlockRequest"
+ },
+ {
+ "id": 45,
+ "name": "initiateMultiPartUploadRequest",
+ "type": "MultipartInfoInitiateRequest"
+ },
+ {
+ "id": 46,
+ "name": "commitMultiPartUploadRequest",
+ "type": "MultipartCommitUploadPartRequest"
+ },
+ {
+ "id": 47,
+ "name": "completeMultiPartUploadRequest",
+ "type": "MultipartUploadCompleteRequest"
+ },
+ {
+ "id": 48,
+ "name": "abortMultiPartUploadRequest",
+ "type": "MultipartUploadAbortRequest"
+ },
+ {
+ "id": 49,
+ "name": "getS3SecretRequest",
+ "type": "GetS3SecretRequest"
+ },
+ {
+ "id": 50,
+ "name": "listMultipartUploadPartsRequest",
+ "type": "MultipartUploadListPartsRequest"
+ },
+ {
+ "id": 51,
+ "name": "serviceListRequest",
+ "type": "ServiceListRequest"
+ },
+ {
+ "id": 53,
+ "name": "dbUpdatesRequest",
+ "type": "DBUpdatesRequest"
+ },
+ {
+ "id": 61,
+ "name": "getDelegationTokenRequest",
+ "type": "hadoop.common.GetDelegationTokenRequestProto"
+ },
+ {
+ "id": 62,
+ "name": "renewDelegationTokenRequest",
+ "type": "hadoop.common.RenewDelegationTokenRequestProto"
+ },
+ {
+ "id": 63,
+ "name": "cancelDelegationTokenRequest",
+ "type": "hadoop.common.CancelDelegationTokenRequestProto"
+ },
+ {
+ "id": 64,
+ "name": "updateGetDelegationTokenRequest",
+ "type": "UpdateGetDelegationTokenRequest"
+ },
+ {
+ "id": 65,
+ "name": "updatedRenewDelegationTokenRequest",
+ "type": "UpdateRenewDelegationTokenRequest"
+ },
+ {
+ "id": 70,
+ "name": "getFileStatusRequest",
+ "type": "GetFileStatusRequest"
+ },
+ {
+ "id": 71,
+ "name": "createDirectoryRequest",
+ "type": "CreateDirectoryRequest"
+ },
+ {
+ "id": 72,
+ "name": "createFileRequest",
+ "type": "CreateFileRequest"
+ },
+ {
+ "id": 73,
+ "name": "lookupFileRequest",
+ "type": "LookupFileRequest"
+ },
+ {
+ "id": 74,
+ "name": "listStatusRequest",
+ "type": "ListStatusRequest"
+ },
+ {
+ "id": 75,
+ "name": "addAclRequest",
+ "type": "AddAclRequest"
+ },
+ {
+ "id": 76,
+ "name": "removeAclRequest",
+ "type": "RemoveAclRequest"
+ },
+ {
+ "id": 77,
+ "name": "setAclRequest",
+ "type": "SetAclRequest"
+ },
+ {
+ "id": 78,
+ "name": "getAclRequest",
+ "type": "GetAclRequest"
+ },
+ {
+ "id": 81,
+ "name": "purgeKeysRequest",
+ "type": "PurgeKeysRequest"
+ },
+ {
+ "id": 82,
+ "name": "updateGetS3SecretRequest",
+ "type": "UpdateGetS3SecretRequest"
+ },
+ {
+ "id": 83,
+ "name": "listMultipartUploadsRequest",
+ "type": "ListMultipartUploadsRequest"
+ },
+ {
+ "id": 91,
+ "name": "listTrashRequest",
+ "type": "ListTrashRequest"
+ },
+ {
+ "id": 92,
+ "name": "RecoverTrashRequest",
+ "type": "RecoverTrashRequest"
+ }
+ ]
+ },
+ {
+ "name": "OMResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "cmdType",
+ "type": "Type"
+ },
+ {
+ "id": 2,
+ "name": "traceID",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "success",
+ "type": "bool",
+ "options": [
+ {
+ "name": "default",
+ "value": "true"
+ }
+ ]
+ },
+ {
+ "id": 4,
+ "name": "message",
+ "type": "string"
+ },
+ {
+ "id": 5,
+ "name": "status",
+ "type": "Status"
+ },
+ {
+ "id": 6,
+ "name": "leaderOMNodeId",
+ "type": "string"
+ },
+ {
+ "id": 11,
+ "name": "createVolumeResponse",
+ "type": "CreateVolumeResponse"
+ },
+ {
+ "id": 12,
+ "name": "setVolumePropertyResponse",
+ "type": "SetVolumePropertyResponse"
+ },
+ {
+ "id": 13,
+ "name": "checkVolumeAccessResponse",
+ "type": "CheckVolumeAccessResponse"
+ },
+ {
+ "id": 14,
+ "name": "infoVolumeResponse",
+ "type": "InfoVolumeResponse"
+ },
+ {
+ "id": 15,
+ "name": "deleteVolumeResponse",
+ "type": "DeleteVolumeResponse"
+ },
+ {
+ "id": 16,
+ "name": "listVolumeResponse",
+ "type": "ListVolumeResponse"
+ },
+ {
+ "id": 21,
+ "name": "createBucketResponse",
+ "type": "CreateBucketResponse"
+ },
+ {
+ "id": 22,
+ "name": "infoBucketResponse",
+ "type": "InfoBucketResponse"
+ },
+ {
+ "id": 23,
+ "name": "setBucketPropertyResponse",
+ "type": "SetBucketPropertyResponse"
+ },
+ {
+ "id": 24,
+ "name": "deleteBucketResponse",
+ "type": "DeleteBucketResponse"
+ },
+ {
+ "id": 25,
+ "name": "listBucketsResponse",
+ "type": "ListBucketsResponse"
+ },
+ {
+ "id": 31,
+ "name": "createKeyResponse",
+ "type": "CreateKeyResponse"
+ },
+ {
+ "id": 32,
+ "name": "lookupKeyResponse",
+ "type": "LookupKeyResponse"
+ },
+ {
+ "id": 33,
+ "name": "renameKeyResponse",
+ "type": "RenameKeyResponse"
+ },
+ {
+ "id": 34,
+ "name": "deleteKeyResponse",
+ "type": "DeleteKeyResponse"
+ },
+ {
+ "id": 35,
+ "name": "listKeysResponse",
+ "type": "ListKeysResponse"
+ },
+ {
+ "id": 36,
+ "name": "commitKeyResponse",
+ "type": "CommitKeyResponse"
+ },
+ {
+ "id": 37,
+ "name": "allocateBlockResponse",
+ "type": "AllocateBlockResponse"
+ },
+ {
+ "id": 45,
+ "name": "initiateMultiPartUploadResponse",
+ "type": "MultipartInfoInitiateResponse"
+ },
+ {
+ "id": 46,
+ "name": "commitMultiPartUploadResponse",
+ "type": "MultipartCommitUploadPartResponse"
+ },
+ {
+ "id": 47,
+ "name": "completeMultiPartUploadResponse",
+ "type": "MultipartUploadCompleteResponse"
+ },
+ {
+ "id": 48,
+ "name": "abortMultiPartUploadResponse",
+ "type": "MultipartUploadAbortResponse"
+ },
+ {
+ "id": 49,
+ "name": "getS3SecretResponse",
+ "type": "GetS3SecretResponse"
+ },
+ {
+ "id": 50,
+ "name": "listMultipartUploadPartsResponse",
+ "type": "MultipartUploadListPartsResponse"
+ },
+ {
+ "id": 51,
+ "name": "ServiceListResponse",
+ "type": "ServiceListResponse"
+ },
+ {
+ "id": 52,
+ "name": "dbUpdatesResponse",
+ "type": "DBUpdatesResponse"
+ },
+ {
+ "id": 61,
+ "name": "getDelegationTokenResponse",
+ "type": "GetDelegationTokenResponseProto"
+ },
+ {
+ "id": 62,
+ "name": "renewDelegationTokenResponse",
+ "type": "RenewDelegationTokenResponseProto"
+ },
+ {
+ "id": 63,
+ "name": "cancelDelegationTokenResponse",
+ "type": "CancelDelegationTokenResponseProto"
+ },
+ {
+ "id": 70,
+ "name": "getFileStatusResponse",
+ "type": "GetFileStatusResponse"
+ },
+ {
+ "id": 71,
+ "name": "createDirectoryResponse",
+ "type": "CreateDirectoryResponse"
+ },
+ {
+ "id": 72,
+ "name": "createFileResponse",
+ "type": "CreateFileResponse"
+ },
+ {
+ "id": 73,
+ "name": "lookupFileResponse",
+ "type": "LookupFileResponse"
+ },
+ {
+ "id": 74,
+ "name": "listStatusResponse",
+ "type": "ListStatusResponse"
+ },
+ {
+ "id": 75,
+ "name": "addAclResponse",
+ "type": "AddAclResponse"
+ },
+ {
+ "id": 76,
+ "name": "removeAclResponse",
+ "type": "RemoveAclResponse"
+ },
+ {
+ "id": 77,
+ "name": "setAclResponse",
+ "type": "SetAclResponse"
+ },
+ {
+ "id": 78,
+ "name": "getAclResponse",
+ "type": "GetAclResponse"
+ },
+ {
+ "id": 81,
+ "name": "purgeKeysResponse",
+ "type": "PurgeKeysResponse"
+ },
+ {
+ "id": 82,
+ "name": "listMultipartUploadsResponse",
+ "type": "ListMultipartUploadsResponse"
+ },
+ {
+ "id": 91,
+ "name": "listTrashResponse",
+ "type": "ListTrashResponse"
+ },
+ {
+ "id": 92,
+ "name": "RecoverTrashResponse",
+ "type": "RecoverTrashResponse"
+ }
+ ]
+ },
+ {
+ "name": "ListTrashRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volumeName",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "bucketName",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "startKeyName",
+ "type": "string"
+ },
+ {
+ "id": 4,
+ "name": "keyPrefix",
+ "type": "string"
+ },
+ {
+ "id": 5,
+ "name": "maxKeys",
+ "type": "int32"
+ }
+ ]
+ },
+ {
+ "name": "ListTrashResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "deletedKeys",
+ "type": "RepeatedKeyInfo",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "RecoverTrashRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volumeName",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "bucketName",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "keyName",
+ "type": "string"
+ },
+ {
+ "id": 4,
+ "name": "destinationBucket",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "RecoverTrashResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "response",
+ "type": "bool"
+ }
+ ]
+ },
+ {
+ "name": "VolumeInfo",
+ "fields": [
+ {
+ "id": 1,
+ "name": "adminName",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "ownerName",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "volume",
+ "type": "string"
+ },
+ {
+ "id": 4,
+ "name": "quotaInBytes",
+ "type": "uint64"
+ },
+ {
+ "id": 5,
+ "name": "metadata",
+ "type": "hadoop.hdds.KeyValue",
+ "is_repeated": true
+ },
+ {
+ "id": 6,
+ "name": "volumeAcls",
+ "type": "OzoneAclInfo",
+ "is_repeated": true
+ },
+ {
+ "id": 7,
+ "name": "creationTime",
+ "type": "uint64"
+ },
+ {
+ "id": 8,
+ "name": "objectID",
+ "type": "uint64"
+ },
+ {
+ "id": 9,
+ "name": "updateID",
+ "type": "uint64"
+ }
+ ]
+ },
+ {
+ "name": "UserInfo",
+ "fields": [
+ {
+ "id": 1,
+ "name": "userName",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "remoteAddress",
+ "type": "string"
+ },
+ {
+ "id": 4,
+ "name": "hostName",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "UpdateGetDelegationTokenRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "getDelegationTokenResponse",
+ "type": "GetDelegationTokenResponseProto"
+ },
+ {
+ "id": 2,
+ "name": "tokenRenewInterval",
+ "type": "uint64"
+ }
+ ]
+ },
+ {
+ "name": "UpdateRenewDelegationTokenRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "renewDelegationTokenRequest",
+ "type": "hadoop.common.RenewDelegationTokenRequestProto"
+ },
+ {
+ "id": 2,
+ "name": "renewDelegationTokenResponse",
+ "type": "RenewDelegationTokenResponseProto"
+ }
+ ]
+ },
+ {
+ "name": "CreateVolumeRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volumeInfo",
+ "type": "VolumeInfo"
+ }
+ ]
+ },
+ {
+ "name": "CreateVolumeResponse"
+ },
+ {
+ "name": "UserVolumeInfo",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volumeNames",
+ "type": "string",
+ "is_repeated": true
+ },
+ {
+ "id": 2,
+ "name": "objectID",
+ "type": "uint64"
+ },
+ {
+ "id": 3,
+ "name": "updateID",
+ "type": "uint64"
+ }
+ ]
+ },
+ {
+ "name": "SetVolumePropertyRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volumeName",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "ownerName",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "quotaInBytes",
+ "type": "uint64"
+ }
+ ]
+ },
+ {
+ "name": "SetVolumePropertyResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "response",
+ "type": "bool"
+ }
+ ]
+ },
+ {
+ "name": "CheckVolumeAccessRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volumeName",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "userAcl",
+ "type": "OzoneAclInfo"
+ }
+ ]
+ },
+ {
+ "name": "CheckVolumeAccessResponse"
+ },
+ {
+ "name": "InfoVolumeRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volumeName",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "InfoVolumeResponse",
+ "fields": [
+ {
+ "id": 2,
+ "name": "volumeInfo",
+ "type": "VolumeInfo"
+ }
+ ]
+ },
+ {
+ "name": "DeleteVolumeRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volumeName",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "DeleteVolumeResponse"
+ },
+ {
+ "name": "ListVolumeRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "scope",
+ "type": "Scope"
+ },
+ {
+ "id": 2,
+ "name": "userName",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "prefix",
+ "type": "string"
+ },
+ {
+ "id": 4,
+ "name": "prevKey",
+ "type": "string"
+ },
+ {
+ "id": 5,
+ "name": "maxKeys",
+ "type": "uint32"
+ }
+ ]
+ },
+ {
+ "name": "ListVolumeResponse",
+ "fields": [
+ {
+ "id": 2,
+ "name": "volumeInfo",
+ "type": "VolumeInfo",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "BucketInfo",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volumeName",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "bucketName",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "acls",
+ "type": "OzoneAclInfo",
+ "is_repeated": true
+ },
+ {
+ "id": 4,
+ "name": "isVersionEnabled",
+ "type": "bool",
+ "options": [
+ {
+ "name": "default",
+ "value": "false"
+ }
+ ]
+ },
+ {
+ "id": 5,
+ "name": "storageType",
+ "type": "StorageTypeProto",
+ "options": [
+ {
+ "name": "default",
+ "value": "DISK"
+ }
+ ]
+ },
+ {
+ "id": 6,
+ "name": "creationTime",
+ "type": "uint64"
+ },
+ {
+ "id": 7,
+ "name": "metadata",
+ "type": "hadoop.hdds.KeyValue",
+ "is_repeated": true
+ },
+ {
+ "id": 8,
+ "name": "beinfo",
+ "type": "BucketEncryptionInfoProto"
+ },
+ {
+ "id": 9,
+ "name": "objectID",
+ "type": "uint64"
+ },
+ {
+ "id": 10,
+ "name": "updateID",
+ "type": "uint64"
+ }
+ ]
+ },
+ {
+ "name": "BucketEncryptionInfoProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "keyName",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "suite",
+ "type": "CipherSuiteProto"
+ },
+ {
+ "id": 3,
+ "name": "cryptoProtocolVersion",
+ "type": "CryptoProtocolVersionProto"
+ }
+ ]
+ },
+ {
+ "name": "FileEncryptionInfoProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "suite",
+ "type": "CipherSuiteProto"
+ },
+ {
+ "id": 2,
+ "name": "cryptoProtocolVersion",
+ "type": "CryptoProtocolVersionProto"
+ },
+ {
+ "id": 3,
+ "name": "key",
+ "type": "bytes"
+ },
+ {
+ "id": 4,
+ "name": "iv",
+ "type": "bytes"
+ },
+ {
+ "id": 5,
+ "name": "keyName",
+ "type": "string"
+ },
+ {
+ "id": 6,
+ "name": "ezKeyVersionName",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "PerFileEncryptionInfoProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "key",
+ "type": "bytes"
+ },
+ {
+ "id": 2,
+ "name": "iv",
+ "type": "bytes"
+ },
+ {
+ "id": 3,
+ "name": "ezKeyVersionName",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "DataEncryptionKeyProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "keyId",
+ "type": "uint32"
+ },
+ {
+ "id": 3,
+ "name": "nonce",
+ "type": "bytes"
+ },
+ {
+ "id": 4,
+ "name": "encryptionKey",
+ "type": "bytes"
+ },
+ {
+ "id": 5,
+ "name": "expiryDate",
+ "type": "uint64"
+ },
+ {
+ "id": 6,
+ "name": "encryptionAlgorithm",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "BucketArgs",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volumeName",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "bucketName",
+ "type": "string"
+ },
+ {
+ "id": 5,
+ "name": "isVersionEnabled",
+ "type": "bool"
+ },
+ {
+ "id": 6,
+ "name": "storageType",
+ "type": "StorageTypeProto"
+ },
+ {
+ "id": 7,
+ "name": "metadata",
+ "type": "hadoop.hdds.KeyValue",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "PrefixInfo",
+ "fields": [
+ {
+ "id": 1,
+ "name": "name",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "acls",
+ "type": "OzoneAclInfo",
+ "is_repeated": true
+ },
+ {
+ "id": 3,
+ "name": "metadata",
+ "type": "hadoop.hdds.KeyValue",
+ "is_repeated": true
+ },
+ {
+ "id": 4,
+ "name": "objectID",
+ "type": "uint64"
+ },
+ {
+ "id": 5,
+ "name": "updateID",
+ "type": "uint64"
+ }
+ ]
+ },
+ {
+ "name": "OzoneObj",
+ "fields": [
+ {
+ "id": 1,
+ "name": "resType",
+ "type": "ObjectType"
+ },
+ {
+ "id": 2,
+ "name": "storeType",
+ "type": "StoreType",
+ "options": [
+ {
+ "name": "default",
+ "value": "S3"
+ }
+ ]
+ },
+ {
+ "id": 3,
+ "name": "path",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "OzoneAclInfo",
+ "fields": [
+ {
+ "id": 1,
+ "name": "type",
+ "type": "OzoneAclType"
+ },
+ {
+ "id": 2,
+ "name": "name",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "rights",
+ "type": "bytes"
+ },
+ {
+ "id": 4,
+ "name": "aclScope",
+ "type": "OzoneAclScope",
+ "options": [
+ {
+ "name": "default",
+ "value": "ACCESS"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "GetAclRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "obj",
+ "type": "OzoneObj"
+ }
+ ]
+ },
+ {
+ "name": "GetAclResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "acls",
+ "type": "OzoneAclInfo",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "AddAclRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "obj",
+ "type": "OzoneObj"
+ },
+ {
+ "id": 2,
+ "name": "acl",
+ "type": "OzoneAclInfo"
+ }
+ ]
+ },
+ {
+ "name": "AddAclResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "response",
+ "type": "bool"
+ }
+ ]
+ },
+ {
+ "name": "RemoveAclRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "obj",
+ "type": "OzoneObj"
+ },
+ {
+ "id": 2,
+ "name": "acl",
+ "type": "OzoneAclInfo"
+ }
+ ]
+ },
+ {
+ "name": "RemoveAclResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "response",
+ "type": "bool"
+ }
+ ]
+ },
+ {
+ "name": "SetAclRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "obj",
+ "type": "OzoneObj"
+ },
+ {
+ "id": 2,
+ "name": "acl",
+ "type": "OzoneAclInfo",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "SetAclResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "response",
+ "type": "bool"
+ }
+ ]
+ },
+ {
+ "name": "CreateBucketRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "bucketInfo",
+ "type": "BucketInfo"
+ }
+ ]
+ },
+ {
+ "name": "CreateBucketResponse"
+ },
+ {
+ "name": "InfoBucketRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volumeName",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "bucketName",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "InfoBucketResponse",
+ "fields": [
+ {
+ "id": 2,
+ "name": "bucketInfo",
+ "type": "BucketInfo"
+ }
+ ]
+ },
+ {
+ "name": "SetBucketPropertyRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "bucketArgs",
+ "type": "BucketArgs"
+ }
+ ]
+ },
+ {
+ "name": "SetBucketPropertyResponse"
+ },
+ {
+ "name": "DeleteBucketRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volumeName",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "bucketName",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "DeleteBucketResponse"
+ },
+ {
+ "name": "ListBucketsRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volumeName",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "startKey",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "prefix",
+ "type": "string"
+ },
+ {
+ "id": 4,
+ "name": "count",
+ "type": "int32"
+ }
+ ]
+ },
+ {
+ "name": "ListBucketsResponse",
+ "fields": [
+ {
+ "id": 2,
+ "name": "bucketInfo",
+ "type": "BucketInfo",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "KeyArgs",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volumeName",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "bucketName",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "keyName",
+ "type": "string"
+ },
+ {
+ "id": 4,
+ "name": "dataSize",
+ "type": "uint64"
+ },
+ {
+ "id": 5,
+ "name": "type",
+ "type": "hadoop.hdds.ReplicationType"
+ },
+ {
+ "id": 6,
+ "name": "factor",
+ "type": "hadoop.hdds.ReplicationFactor"
+ },
+ {
+ "id": 7,
+ "name": "keyLocations",
+ "type": "KeyLocation",
+ "is_repeated": true
+ },
+ {
+ "id": 8,
+ "name": "isMultipartKey",
+ "type": "bool"
+ },
+ {
+ "id": 9,
+ "name": "multipartUploadID",
+ "type": "string"
+ },
+ {
+ "id": 10,
+ "name": "multipartNumber",
+ "type": "uint32"
+ },
+ {
+ "id": 11,
+ "name": "metadata",
+ "type": "hadoop.hdds.KeyValue",
+ "is_repeated": true
+ },
+ {
+ "id": 12,
+ "name": "acls",
+ "type": "OzoneAclInfo",
+ "is_repeated": true
+ },
+ {
+ "id": 13,
+ "name": "modificationTime",
+ "type": "uint64"
+ },
+ {
+ "id": 14,
+ "name": "sortDatanodes",
+ "type": "bool"
+ },
+ {
+ "id": 15,
+ "name": "fileEncryptionInfo",
+ "type": "FileEncryptionInfoProto"
+ }
+ ]
+ },
+ {
+ "name": "KeyLocation",
+ "fields": [
+ {
+ "id": 1,
+ "name": "blockID",
+ "type": "hadoop.hdds.BlockID"
+ },
+ {
+ "id": 3,
+ "name": "offset",
+ "type": "uint64"
+ },
+ {
+ "id": 4,
+ "name": "length",
+ "type": "uint64"
+ },
+ {
+ "id": 5,
+ "name": "createVersion",
+ "type": "uint64"
+ },
+ {
+ "id": 6,
+ "name": "token",
+ "type": "hadoop.common.TokenProto"
+ },
+ {
+ "id": 7,
+ "name": "pipeline",
+ "type": "hadoop.hdds.Pipeline"
+ }
+ ]
+ },
+ {
+ "name": "KeyLocationList",
+ "fields": [
+ {
+ "id": 1,
+ "name": "version",
+ "type": "uint64"
+ },
+ {
+ "id": 2,
+ "name": "keyLocations",
+ "type": "KeyLocation",
+ "is_repeated": true
+ },
+ {
+ "id": 3,
+ "name": "fileEncryptionInfo",
+ "type": "FileEncryptionInfoProto"
+ }
+ ]
+ },
+ {
+ "name": "KeyInfo",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volumeName",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "bucketName",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "keyName",
+ "type": "string"
+ },
+ {
+ "id": 4,
+ "name": "dataSize",
+ "type": "uint64"
+ },
+ {
+ "id": 5,
+ "name": "type",
+ "type": "hadoop.hdds.ReplicationType"
+ },
+ {
+ "id": 6,
+ "name": "factor",
+ "type": "hadoop.hdds.ReplicationFactor"
+ },
+ {
+ "id": 7,
+ "name": "keyLocationList",
+ "type": "KeyLocationList",
+ "is_repeated": true
+ },
+ {
+ "id": 8,
+ "name": "creationTime",
+ "type": "uint64"
+ },
+ {
+ "id": 9,
+ "name": "modificationTime",
+ "type": "uint64"
+ },
+ {
+ "id": 10,
+ "name": "latestVersion",
+ "type": "uint64"
+ },
+ {
+ "id": 11,
+ "name": "metadata",
+ "type": "hadoop.hdds.KeyValue",
+ "is_repeated": true
+ },
+ {
+ "id": 12,
+ "name": "fileEncryptionInfo",
+ "type": "FileEncryptionInfoProto"
+ },
+ {
+ "id": 13,
+ "name": "acls",
+ "type": "OzoneAclInfo",
+ "is_repeated": true
+ },
+ {
+ "id": 14,
+ "name": "objectID",
+ "type": "uint64"
+ },
+ {
+ "id": 15,
+ "name": "updateID",
+ "type": "uint64"
+ }
+ ]
+ },
+ {
+ "name": "RepeatedKeyInfo",
+ "fields": [
+ {
+ "id": 1,
+ "name": "keyInfo",
+ "type": "KeyInfo",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "OzoneFileStatusProto",
+ "fields": [
+ {
+ "id": 2,
+ "name": "keyInfo",
+ "type": "KeyInfo"
+ },
+ {
+ "id": 3,
+ "name": "blockSize",
+ "type": "uint64"
+ },
+ {
+ "id": 4,
+ "name": "isDirectory",
+ "type": "bool"
+ }
+ ]
+ },
+ {
+ "name": "GetFileStatusRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "keyArgs",
+ "type": "KeyArgs"
+ }
+ ]
+ },
+ {
+ "name": "GetFileStatusResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "status",
+ "type": "OzoneFileStatusProto"
+ }
+ ]
+ },
+ {
+ "name": "CreateDirectoryRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "keyArgs",
+ "type": "KeyArgs"
+ }
+ ]
+ },
+ {
+ "name": "CreateDirectoryResponse"
+ },
+ {
+ "name": "CreateFileRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "keyArgs",
+ "type": "KeyArgs"
+ },
+ {
+ "id": 2,
+ "name": "isRecursive",
+ "type": "bool"
+ },
+ {
+ "id": 3,
+ "name": "isOverwrite",
+ "type": "bool"
+ },
+ {
+ "id": 4,
+ "name": "clientID",
+ "type": "uint64"
+ }
+ ]
+ },
+ {
+ "name": "CreateFileResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "keyInfo",
+ "type": "KeyInfo"
+ },
+ {
+ "id": 2,
+ "name": "ID",
+ "type": "uint64"
+ },
+ {
+ "id": 3,
+ "name": "openVersion",
+ "type": "uint64"
+ }
+ ]
+ },
+ {
+ "name": "LookupFileRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "keyArgs",
+ "type": "KeyArgs"
+ }
+ ]
+ },
+ {
+ "name": "LookupFileResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "keyInfo",
+ "type": "KeyInfo"
+ }
+ ]
+ },
+ {
+ "name": "ListStatusRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "keyArgs",
+ "type": "KeyArgs"
+ },
+ {
+ "id": 2,
+ "name": "recursive",
+ "type": "bool"
+ },
+ {
+ "id": 3,
+ "name": "startKey",
+ "type": "string"
+ },
+ {
+ "id": 4,
+ "name": "numEntries",
+ "type": "uint64"
+ }
+ ]
+ },
+ {
+ "name": "ListStatusResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "statuses",
+ "type": "OzoneFileStatusProto",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "CreateKeyRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "keyArgs",
+ "type": "KeyArgs"
+ },
+ {
+ "id": 2,
+ "name": "clientID",
+ "type": "uint64"
+ }
+ ]
+ },
+ {
+ "name": "CreateKeyResponse",
+ "fields": [
+ {
+ "id": 2,
+ "name": "keyInfo",
+ "type": "KeyInfo"
+ },
+ {
+ "id": 3,
+ "name": "ID",
+ "type": "uint64"
+ },
+ {
+ "id": 4,
+ "name": "openVersion",
+ "type": "uint64"
+ }
+ ]
+ },
+ {
+ "name": "LookupKeyRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "keyArgs",
+ "type": "KeyArgs"
+ }
+ ]
+ },
+ {
+ "name": "LookupKeyResponse",
+ "fields": [
+ {
+ "id": 2,
+ "name": "keyInfo",
+ "type": "KeyInfo"
+ },
+ {
+ "id": 3,
+ "name": "ID",
+ "type": "uint64"
+ },
+ {
+ "id": 4,
+ "name": "openVersion",
+ "type": "uint64"
+ }
+ ]
+ },
+ {
+ "name": "RenameKeyRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "keyArgs",
+ "type": "KeyArgs"
+ },
+ {
+ "id": 2,
+ "name": "toKeyName",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "RenameKeyResponse"
+ },
+ {
+ "name": "DeleteKeyRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "keyArgs",
+ "type": "KeyArgs"
+ }
+ ]
+ },
+ {
+ "name": "DeleteKeyResponse",
+ "fields": [
+ {
+ "id": 2,
+ "name": "keyInfo",
+ "type": "KeyInfo"
+ },
+ {
+ "id": 3,
+ "name": "ID",
+ "type": "uint64"
+ },
+ {
+ "id": 4,
+ "name": "openVersion",
+ "type": "uint64"
+ }
+ ]
+ },
+ {
+ "name": "DeletedKeys",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volumeName",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "bucketName",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "keys",
+ "type": "string",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "PurgeKeysRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "deletedKeys",
+ "type": "DeletedKeys",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "PurgeKeysResponse"
+ },
+ {
+ "name": "OMTokenProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "type",
+ "type": "Type"
+ },
+ {
+ "id": 2,
+ "name": "version",
+ "type": "uint32"
+ },
+ {
+ "id": 3,
+ "name": "owner",
+ "type": "string"
+ },
+ {
+ "id": 4,
+ "name": "renewer",
+ "type": "string"
+ },
+ {
+ "id": 5,
+ "name": "realUser",
+ "type": "string"
+ },
+ {
+ "id": 6,
+ "name": "issueDate",
+ "type": "uint64"
+ },
+ {
+ "id": 7,
+ "name": "maxDate",
+ "type": "uint64"
+ },
+ {
+ "id": 8,
+ "name": "sequenceNumber",
+ "type": "uint32"
+ },
+ {
+ "id": 9,
+ "name": "masterKeyId",
+ "type": "uint32"
+ },
+ {
+ "id": 10,
+ "name": "expiryDate",
+ "type": "uint64"
+ },
+ {
+ "id": 11,
+ "name": "omCertSerialId",
+ "type": "string"
+ },
+ {
+ "id": 12,
+ "name": "accessKeyId",
+ "type": "string"
+ },
+ {
+ "id": 13,
+ "name": "signature",
+ "type": "string"
+ },
+ {
+ "id": 14,
+ "name": "strToSign",
+ "type": "string"
+ },
+ {
+ "id": 15,
+ "name": "omServiceId",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "SecretKeyProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "keyId",
+ "type": "uint32"
+ },
+ {
+ "id": 2,
+ "name": "expiryDate",
+ "type": "uint64"
+ },
+ {
+ "id": 3,
+ "name": "privateKeyBytes",
+ "type": "bytes"
+ },
+ {
+ "id": 4,
+ "name": "publicKeyBytes",
+ "type": "bytes"
+ }
+ ]
+ },
+ {
+ "name": "ListKeysRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volumeName",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "bucketName",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "startKey",
+ "type": "string"
+ },
+ {
+ "id": 4,
+ "name": "prefix",
+ "type": "string"
+ },
+ {
+ "id": 5,
+ "name": "count",
+ "type": "int32"
+ }
+ ]
+ },
+ {
+ "name": "ListKeysResponse",
+ "fields": [
+ {
+ "id": 2,
+ "name": "keyInfo",
+ "type": "KeyInfo",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "CommitKeyRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "keyArgs",
+ "type": "KeyArgs"
+ },
+ {
+ "id": 2,
+ "name": "clientID",
+ "type": "uint64"
+ }
+ ]
+ },
+ {
+ "name": "CommitKeyResponse"
+ },
+ {
+ "name": "AllocateBlockRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "keyArgs",
+ "type": "KeyArgs"
+ },
+ {
+ "id": 2,
+ "name": "clientID",
+ "type": "uint64"
+ },
+ {
+ "id": 3,
+ "name": "excludeList",
+ "type": "hadoop.hdds.ExcludeListProto"
+ },
+ {
+ "id": 4,
+ "name": "keyLocation",
+ "type": "KeyLocation"
+ }
+ ]
+ },
+ {
+ "name": "AllocateBlockResponse",
+ "fields": [
+ {
+ "id": 2,
+ "name": "keyLocation",
+ "type": "KeyLocation"
+ }
+ ]
+ },
+ {
+ "name": "ServiceListRequest"
+ },
+ {
+ "name": "DBUpdatesRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "sequenceNumber",
+ "type": "uint64"
+ }
+ ]
+ },
+ {
+ "name": "ServiceListResponse",
+ "fields": [
+ {
+ "id": 2,
+ "name": "serviceInfo",
+ "type": "ServiceInfo",
+ "is_repeated": true
+ },
+ {
+ "id": 3,
+ "name": "caCertificate",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "DBUpdatesResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "sequenceNumber",
+ "type": "uint64"
+ },
+ {
+ "id": 2,
+ "name": "data",
+ "type": "bytes",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "ServicePort",
+ "fields": [
+ {
+ "id": 1,
+ "name": "type",
+ "type": "Type"
+ },
+ {
+ "id": 2,
+ "name": "value",
+ "type": "uint32"
+ }
+ ]
+ },
+ {
+ "name": "OMRoleInfo",
+ "fields": [
+ {
+ "id": 1,
+ "name": "nodeId",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "serverRole",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "ServiceInfo",
+ "fields": [
+ {
+ "id": 1,
+ "name": "nodeType",
+ "type": "hadoop.hdds.NodeType"
+ },
+ {
+ "id": 2,
+ "name": "hostname",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "servicePorts",
+ "type": "ServicePort",
+ "is_repeated": true
+ },
+ {
+ "id": 4,
+ "name": "omRole",
+ "type": "OMRoleInfo"
+ }
+ ]
+ },
+ {
+ "name": "MultipartInfoInitiateRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "keyArgs",
+ "type": "KeyArgs"
+ }
+ ]
+ },
+ {
+ "name": "MultipartInfoInitiateResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volumeName",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "bucketName",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "keyName",
+ "type": "string"
+ },
+ {
+ "id": 4,
+ "name": "multipartUploadID",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "MultipartKeyInfo",
+ "fields": [
+ {
+ "id": 1,
+ "name": "uploadID",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "creationTime",
+ "type": "uint64"
+ },
+ {
+ "id": 3,
+ "name": "type",
+ "type": "hadoop.hdds.ReplicationType"
+ },
+ {
+ "id": 4,
+ "name": "factor",
+ "type": "hadoop.hdds.ReplicationFactor"
+ },
+ {
+ "id": 5,
+ "name": "partKeyInfoList",
+ "type": "PartKeyInfo",
+ "is_repeated": true
+ },
+ {
+ "id": 6,
+ "name": "objectID",
+ "type": "uint64"
+ },
+ {
+ "id": 7,
+ "name": "updateID",
+ "type": "uint64"
+ }
+ ]
+ },
+ {
+ "name": "PartKeyInfo",
+ "fields": [
+ {
+ "id": 1,
+ "name": "partName",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "partNumber",
+ "type": "uint32"
+ },
+ {
+ "id": 3,
+ "name": "partKeyInfo",
+ "type": "KeyInfo"
+ }
+ ]
+ },
+ {
+ "name": "MultipartCommitUploadPartRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "keyArgs",
+ "type": "KeyArgs"
+ },
+ {
+ "id": 2,
+ "name": "clientID",
+ "type": "uint64"
+ }
+ ]
+ },
+ {
+ "name": "MultipartCommitUploadPartResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "partName",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "MultipartUploadCompleteRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "keyArgs",
+ "type": "KeyArgs"
+ },
+ {
+ "id": 2,
+ "name": "partsList",
+ "type": "Part",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "MultipartUploadCompleteResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volume",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "bucket",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "key",
+ "type": "string"
+ },
+ {
+ "id": 4,
+ "name": "hash",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "Part",
+ "fields": [
+ {
+ "id": 1,
+ "name": "partNumber",
+ "type": "uint32"
+ },
+ {
+ "id": 2,
+ "name": "partName",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "MultipartUploadAbortRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "keyArgs",
+ "type": "KeyArgs"
+ }
+ ]
+ },
+ {
+ "name": "MultipartUploadAbortResponse"
+ },
+ {
+ "name": "MultipartUploadListPartsRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volume",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "bucket",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "key",
+ "type": "string"
+ },
+ {
+ "id": 4,
+ "name": "uploadID",
+ "type": "string"
+ },
+ {
+ "id": 5,
+ "name": "partNumbermarker",
+ "type": "uint32"
+ },
+ {
+ "id": 6,
+ "name": "maxParts",
+ "type": "uint32"
+ }
+ ]
+ },
+ {
+ "name": "MultipartUploadListPartsResponse",
+ "fields": [
+ {
+ "id": 2,
+ "name": "type",
+ "type": "hadoop.hdds.ReplicationType"
+ },
+ {
+ "id": 3,
+ "name": "factor",
+ "type": "hadoop.hdds.ReplicationFactor"
+ },
+ {
+ "id": 4,
+ "name": "nextPartNumberMarker",
+ "type": "uint32"
+ },
+ {
+ "id": 5,
+ "name": "isTruncated",
+ "type": "bool"
+ },
+ {
+ "id": 6,
+ "name": "partsList",
+ "type": "PartInfo",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "ListMultipartUploadsRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volume",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "bucket",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "prefix",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "ListMultipartUploadsResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "isTruncated",
+ "type": "bool"
+ },
+ {
+ "id": 2,
+ "name": "uploadsList",
+ "type": "MultipartUploadInfo",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "MultipartUploadInfo",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volumeName",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "bucketName",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "keyName",
+ "type": "string"
+ },
+ {
+ "id": 4,
+ "name": "uploadId",
+ "type": "string"
+ },
+ {
+ "id": 5,
+ "name": "creationTime",
+ "type": "uint64"
+ },
+ {
+ "id": 6,
+ "name": "type",
+ "type": "hadoop.hdds.ReplicationType"
+ },
+ {
+ "id": 7,
+ "name": "factor",
+ "type": "hadoop.hdds.ReplicationFactor"
+ }
+ ]
+ },
+ {
+ "name": "PartInfo",
+ "fields": [
+ {
+ "id": 1,
+ "name": "partNumber",
+ "type": "uint32"
+ },
+ {
+ "id": 2,
+ "name": "partName",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "modificationTime",
+ "type": "uint64"
+ },
+ {
+ "id": 4,
+ "name": "size",
+ "type": "uint64"
+ }
+ ]
+ },
+ {
+ "name": "GetDelegationTokenResponseProto",
+ "fields": [
+ {
+ "id": 2,
+ "name": "response",
+ "type": "hadoop.common.GetDelegationTokenResponseProto"
+ }
+ ]
+ },
+ {
+ "name": "RenewDelegationTokenResponseProto",
+ "fields": [
+ {
+ "id": 2,
+ "name": "response",
+ "type": "hadoop.common.RenewDelegationTokenResponseProto"
+ }
+ ]
+ },
+ {
+ "name": "CancelDelegationTokenResponseProto",
+ "fields": [
+ {
+ "id": 2,
+ "name": "response",
+ "type": "hadoop.common.CancelDelegationTokenResponseProto"
+ }
+ ]
+ },
+ {
+ "name": "S3Secret",
+ "fields": [
+ {
+ "id": 1,
+ "name": "kerberosID",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "awsSecret",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "GetS3SecretRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "kerberosID",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "GetS3SecretResponse",
+ "fields": [
+ {
+ "id": 2,
+ "name": "s3Secret",
+ "type": "S3Secret"
+ }
+ ]
+ },
+ {
+ "name": "UpdateGetS3SecretRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "kerberosID",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "awsSecret",
+ "type": "string"
+ }
+ ]
+ }
+ ],
+ "services": [
+ {
+ "name": "OzoneManagerService",
+ "rpcs": [
+ {
+ "name": "submitRequest",
+ "in_type": "OMRequest",
+ "out_type": "OMResponse"
+ }
+ ]
+ }
+ ],
+ "imports": [
+ {
+ "path": "hdds.proto"
+ },
+ {
+ "path": "Security.proto"
+ }
+ ],
+ "package": {
+ "name": "hadoop.ozone"
+ },
+ "options": [
+ {
+ "name": "java_package",
+ "value": "org.apache.hadoop.ozone.protocol.proto"
+ },
+ {
+ "name": "java_outer_classname",
+ "value": "OzoneManagerProtocolProtos"
+ },
+ {
+ "name": "java_generic_services",
+ "value": "true"
+ },
+ {
+ "name": "java_generate_equals_and_hash",
+ "value": "true"
+ }
+ ]
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/TestOmUtils.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/TestOmUtils.java
index cdbb786..ab9b4e1 100644
--- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/TestOmUtils.java
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/TestOmUtils.java
@@ -18,32 +18,23 @@
package org.apache.hadoop.ozone;
-import org.apache.commons.io.FileUtils;
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.List;
+import java.util.Map;
+
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
-import org.apache.hadoop.io.IOUtils;
+
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.rules.Timeout;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assume.assumeTrue;
-
/**
* Unit tests for {@link OmUtils}.
*/
@@ -55,36 +46,6 @@
@Rule
public Timeout timeout = new Timeout(60_000);
- @Test
- public void testWriteCheckpointToOutputStream() throws Exception {
-
- FileInputStream fis = null;
- FileOutputStream fos = null;
-
- try {
- String testDirName = folder.newFolder().getAbsolutePath();
- File file = new File(testDirName + "/temp1.txt");
- FileWriter writer = new FileWriter(file);
- writer.write("Test data 1");
- writer.close();
-
- file = new File(testDirName + "/temp2.txt");
- writer = new FileWriter(file);
- writer.write("Test data 2");
- writer.close();
-
- File outputFile =
- new File(Paths.get(testDirName, "output_file.tgz").toString());
- TestDBCheckpoint dbCheckpoint = new TestDBCheckpoint(
- Paths.get(testDirName));
- OmUtils.writeOmDBCheckpointToStream(dbCheckpoint,
- new FileOutputStream(outputFile));
- assertNotNull(outputFile);
- } finally {
- IOUtils.closeStream(fis);
- IOUtils.closeStream(fos);
- }
- }
@Test
public void createOMDirCreatesDirectoryIfNecessary() throws IOException {
@@ -140,54 +101,3 @@
}
}
-class TestDBCheckpoint implements DBCheckpoint {
-
- private Path checkpointFile;
-
- TestDBCheckpoint(Path checkpointFile) {
- this.checkpointFile = checkpointFile;
- }
-
- @Override
- public Path getCheckpointLocation() {
- return checkpointFile;
- }
-
- @Override
- public long getCheckpointTimestamp() {
- return 0;
- }
-
- @Override
- public long getLatestSequenceNumber() {
- return 0;
- }
-
- @Override
- public long checkpointCreationTimeTaken() {
- return 0;
- }
-
- @Override
- public void cleanupCheckpoint() throws IOException {
- FileUtils.deleteDirectory(checkpointFile.toFile());
- }
-
- @Override
- public void setRatisSnapshotIndex(long omRatisSnapshotIndex) {
- }
-
- @Override
- public long getRatisSnapshotIndex() {
- return 0;
- }
-
- @Override
- public void setRatisSnapshotTerm(long omRatisSnapshotTermIndex) {
- }
-
- @Override
- public long getRatisSnapshotTerm() {
- return 0;
- }
-}
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/ha/TestOMFailoverProxyProvider.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/ha/TestOMFailoverProxyProvider.java
new file mode 100644
index 0000000..e6c6af0
--- /dev/null
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/ha/TestOMFailoverProxyProvider.java
@@ -0,0 +1,195 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.om.ha;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.StringJoiner;
+
+import org.apache.hadoop.io.Text;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.Before;
+
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_NODES_KEY;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.
+ OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_KEY;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.
+ OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_DEFAULT;
+
+/**
+ * Tests OMFailoverProxyProvider failover behaviour.
+ */
+public class TestOMFailoverProxyProvider {
+ private final static String OM_SERVICE_ID = "om-service-test1";
+ private final static String NODE_ID_BASE_STR = "omNode-";
+ private final static String DUMMY_NODE_ADDR = "0.0.0.0:8080";
+ private OMFailoverProxyProvider provider;
+ private long waitBetweenRetries;
+ private int numNodes = 3;
+ private OzoneConfiguration config;
+
+ @Before
+ public void init() throws Exception {
+ config = new OzoneConfiguration();
+ waitBetweenRetries = config.getLong(
+ OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_KEY,
+ OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_DEFAULT);
+ StringJoiner allNodeIds = new StringJoiner(",");
+ for (int i = 1; i <= numNodes; i++) {
+ String nodeId = NODE_ID_BASE_STR + i;
+ config.set(OmUtils.addKeySuffixes(OZONE_OM_ADDRESS_KEY, OM_SERVICE_ID,
+ nodeId), DUMMY_NODE_ADDR);
+ allNodeIds.add(nodeId);
+ }
+ config.set(OmUtils.addKeySuffixes(OZONE_OM_NODES_KEY, OM_SERVICE_ID),
+ allNodeIds.toString());
+ provider = new OMFailoverProxyProvider(config,
+ UserGroupInformation.getCurrentUser(), OM_SERVICE_ID);
+ }
+
+ /**
+ * Tests waitTime when fail over to next node.
+ */
+ @Test
+ public void testWaitTimeWithNextNode() {
+ failoverToNextNode(numNodes - 1, 0);
+ // After 3 attempts done, wait time should be waitBetweenRetries.
+ failoverToNextNode(1, waitBetweenRetries);
+ // From 4th Attempt waitTime should reset to 0.
+ failoverToNextNode(numNodes - 1, 0);
+ // After 2nd round of 3attempts done, wait time should be
+ // waitBetweenRetries.
+ failoverToNextNode(1, waitBetweenRetries);
+ }
+
+ /**
+ * Tests failover to next node and same node.
+ */
+ @Test
+ public void testWaitTimeWithNextNodeAndSameNodeFailover() {
+ failoverToNextNode(1, 0);
+ // 1 Failover attempt to same OM, waitTime should increase.
+ failoverToSameNode(2);
+ }
+
+ /**
+ * Tests wait time should reset in the following case:
+ * 1. Do a couple same node failover attempts.
+ * 2. Next node failover should reset wait time to 0.
+ */
+ @Test
+ public void testWaitTimeResetWhenNextNodeFailoverAfterSameNode() {
+ // 2 failover attempts to same OM, waitTime should increase.
+ failoverToSameNode(2);
+ // Failover to next node, should reset waitTime to 0.
+ failoverToNextNode(1, 0);
+ }
+
+ /**
+ * Tests wait time should be 0 in the following case:
+ * 1. Do failover to suggest new node.
+ * 2. WaitTime should be 0.
+ */
+ @Test
+ public void testWaitTimeWithSuggestedNewNode() {
+ Collection<String> allNodeIds = config.getTrimmedStringCollection(OmUtils.
+ addKeySuffixes(OZONE_OM_NODES_KEY, OM_SERVICE_ID));
+ allNodeIds.remove(provider.getCurrentProxyOMNodeId());
+ Assert.assertTrue("This test needs at least 2 OMs",
+ allNodeIds.size() > 0);
+ provider.performFailoverIfRequired(allNodeIds.iterator().next());
+ Assert.assertEquals(0, provider.getWaitTime());
+ }
+
+ /**
+ * Tests waitTime reset after same node failover.
+ */
+ @Test
+ public void testWaitTimeResetWhenAllNodeFailoverAndSameNode() {
+ // Next node failover wait time should be 0.
+ failoverToNextNode(numNodes - 1, 0);
+ // Once all numNodes failover done, waitTime should be waitBetweenRetries
+ failoverToNextNode(1, waitBetweenRetries);
+ // 4 failover attempts to same OM, waitTime should increase.
+ failoverToSameNode(4);
+ // Next node failover should reset wait time.
+ failoverToNextNode(numNodes - 1, 0);
+ failoverToNextNode(1, waitBetweenRetries);
+ }
+
+ /**
+ * Failover to next node and wait time should be same as waitTimeAfter.
+ */
+ private void failoverToNextNode(int numNextNodeFailoverTimes,
+ long waitTimeAfter) {
+ for (int attempt = 0; attempt < numNextNodeFailoverTimes; attempt++) {
+ provider.performFailoverToNextProxy();
+ Assert.assertEquals(waitTimeAfter, provider.getWaitTime());
+ }
+ }
+
+ /**
+ * Failover to same node and wait time will be attempt*waitBetweenRetries.
+ */
+ private void failoverToSameNode(int numSameNodeFailoverTimes) {
+ for (int attempt = 1; attempt <= numSameNodeFailoverTimes; attempt++) {
+ provider.performFailoverIfRequired(provider.getCurrentProxyOMNodeId());
+ Assert.assertEquals(attempt * waitBetweenRetries,
+ provider.getWaitTime());
+ }
+ }
+
+ /**
+ * Tests canonical delegation token service name in is consistently ordered.
+ */
+ @Test
+ public void testCanonicalTokenServiceName() throws IOException {
+ OzoneConfiguration ozoneConf = new OzoneConfiguration();
+ ArrayList<String> nodeAddrs = new ArrayList<>(
+ Arrays.asList("4.3.2.1:9862", "2.1.0.5:9862", "3.2.1.0:9862"));
+ Assert.assertEquals(numNodes, nodeAddrs.size());
+
+ StringJoiner allNodeIds = new StringJoiner(",");
+ for (int i = 1; i <= numNodes; i++) {
+ String nodeId = NODE_ID_BASE_STR + i;
+ ozoneConf.set(OmUtils.addKeySuffixes(OZONE_OM_ADDRESS_KEY, OM_SERVICE_ID,
+ nodeId), nodeAddrs.get(i-1));
+ allNodeIds.add(nodeId);
+ }
+ ozoneConf.set(OmUtils.addKeySuffixes(OZONE_OM_NODES_KEY, OM_SERVICE_ID),
+ allNodeIds.toString());
+ OMFailoverProxyProvider prov = new OMFailoverProxyProvider(ozoneConf,
+ UserGroupInformation.getCurrentUser(), OM_SERVICE_ID);
+
+ Text dtService = prov.getCurrentProxyDelegationToken();
+
+ Collections.sort(nodeAddrs);
+ String expectedDtService = String.join(",", nodeAddrs);
+ Assert.assertEquals(expectedDtService, dtService.toString());
+ }
+
+}
diff --git a/hadoop-ozone/csi/pom.xml b/hadoop-ozone/csi/pom.xml
index c6eaf3d..9fd2126 100644
--- a/hadoop-ozone/csi/pom.xml
+++ b/hadoop-ozone/csi/pom.xml
@@ -28,15 +28,11 @@
<name>Apache Hadoop Ozone CSI service</name>
<packaging>jar</packaging>
- <properties>
- <grpc-compile.version>1.17.1</grpc-compile.version>
- <protobuf-compile.version>3.5.0</protobuf-compile.version>
- </properties>
<dependencies>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java-util</artifactId>
- <version>3.5.1</version>
+ <version>${protobuf-compile.version}</version>
<exclusions>
<exclusion>
<groupId>com.google.protobuf</groupId>
@@ -66,7 +62,7 @@
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
- <version>3.5.1</version>
+ <version>${protobuf-compile.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
@@ -76,12 +72,20 @@
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport-native-epoll</artifactId>
- <version>4.1.30.Final</version>
+ <version>${netty.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport-native-unix-common</artifactId>
- <version>4.1.30.Final</version>
+ <version>${netty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
@@ -134,6 +138,10 @@
</extensions>
<plugins>
<plugin>
+ <groupId>com.salesforce.servicelibs</groupId>
+ <artifactId>proto-backwards-compatibility</artifactId>
+ </plugin>
+ <plugin>
<groupId>org.xolstice.maven.plugins</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
<version>${protobuf-maven-plugin.version}</version>
diff --git a/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/ControllerService.java b/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/ControllerService.java
index 65b7250..6a6b985 100644
--- a/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/ControllerService.java
+++ b/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/ControllerService.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -43,15 +43,11 @@
*/
public class ControllerService extends ControllerImplBase {
- private final String volumeOwner;
-
private long defaultVolumeSize;
private OzoneClient ozoneClient;
- public ControllerService(OzoneClient ozoneClient, long volumeSize,
- String volumeOwner) {
- this.volumeOwner = volumeOwner;
+ public ControllerService(OzoneClient ozoneClient, long volumeSize) {
this.defaultVolumeSize = volumeSize;
this.ozoneClient = ozoneClient;
}
@@ -60,8 +56,7 @@
public void createVolume(CreateVolumeRequest request,
StreamObserver<CreateVolumeResponse> responseObserver) {
try {
- ozoneClient.getObjectStore()
- .createS3Bucket(volumeOwner, request.getName());
+ ozoneClient.getObjectStore().createS3Bucket(request.getName());
long size = findSize(request.getCapacityRange());
diff --git a/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/CsiServer.java b/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/CsiServer.java
index 443008a..3cd012e 100644
--- a/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/CsiServer.java
+++ b/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/CsiServer.java
@@ -47,7 +47,7 @@
mixinStandardHelpOptions = true)
public class CsiServer extends GenericCli implements Callable<Void> {
- private final static Logger LOG = LoggerFactory.getLogger(CsiServer.class);
+ private static final Logger LOG = LoggerFactory.getLogger(CsiServer.class);
@Override
public Void call() throws Exception {
@@ -73,7 +73,7 @@
.bossEventLoopGroup(group)
.addService(new IdentitiyService())
.addService(new ControllerService(rpcClient,
- csiConfig.getDefaultVolumeSize(), csiConfig.getVolumeOwner()))
+ csiConfig.getDefaultVolumeSize()))
.addService(new NodeService(csiConfig))
.build();
@@ -112,8 +112,7 @@
@Config(key = "s3g.address",
defaultValue = "http://localhost:9878",
description =
- "The defaul t size of the created volumes (if not specified in the"
- + " requests).",
+ "The address of S3 Gateway endpoint.",
tags = ConfigTag.STORAGE)
private String s3gAddress;
diff --git a/hadoop-ozone/csi/src/main/proto/proto.lock b/hadoop-ozone/csi/src/main/proto/proto.lock
new file mode 100644
index 0000000..8f797b6
--- /dev/null
+++ b/hadoop-ozone/csi/src/main/proto/proto.lock
@@ -0,0 +1,1471 @@
+{
+ "definitions": [
+ {
+ "protopath": "csi.proto",
+ "def": {
+ "enums": [
+ {
+ "name": "Service.Type",
+ "enum_fields": [
+ {
+ "name": "UNKNOWN"
+ },
+ {
+ "name": "CONTROLLER_SERVICE",
+ "integer": 1
+ },
+ {
+ "name": "VOLUME_ACCESSIBILITY_CONSTRAINTS",
+ "integer": 2
+ }
+ ]
+ },
+ {
+ "name": "VolumeExpansion.Type",
+ "enum_fields": [
+ {
+ "name": "UNKNOWN"
+ },
+ {
+ "name": "ONLINE",
+ "integer": 1
+ },
+ {
+ "name": "OFFLINE",
+ "integer": 2
+ }
+ ]
+ },
+ {
+ "name": "AccessMode.Mode",
+ "enum_fields": [
+ {
+ "name": "UNKNOWN"
+ },
+ {
+ "name": "SINGLE_NODE_WRITER",
+ "integer": 1
+ },
+ {
+ "name": "SINGLE_NODE_READER_ONLY",
+ "integer": 2
+ },
+ {
+ "name": "MULTI_NODE_READER_ONLY",
+ "integer": 3
+ },
+ {
+ "name": "MULTI_NODE_SINGLE_WRITER",
+ "integer": 4
+ },
+ {
+ "name": "MULTI_NODE_MULTI_WRITER",
+ "integer": 5
+ }
+ ]
+ },
+ {
+ "name": "RPC.Type",
+ "enum_fields": [
+ {
+ "name": "UNKNOWN"
+ },
+ {
+ "name": "CREATE_DELETE_VOLUME",
+ "integer": 1
+ },
+ {
+ "name": "PUBLISH_UNPUBLISH_VOLUME",
+ "integer": 2
+ },
+ {
+ "name": "LIST_VOLUMES",
+ "integer": 3
+ },
+ {
+ "name": "GET_CAPACITY",
+ "integer": 4
+ },
+ {
+ "name": "CREATE_DELETE_SNAPSHOT",
+ "integer": 5
+ },
+ {
+ "name": "LIST_SNAPSHOTS",
+ "integer": 6
+ },
+ {
+ "name": "CLONE_VOLUME",
+ "integer": 7
+ },
+ {
+ "name": "PUBLISH_READONLY",
+ "integer": 8
+ },
+ {
+ "name": "EXPAND_VOLUME",
+ "integer": 9
+ }
+ ]
+ },
+ {
+ "name": "VolumeUsage.Unit",
+ "enum_fields": [
+ {
+ "name": "UNKNOWN"
+ },
+ {
+ "name": "BYTES",
+ "integer": 1
+ },
+ {
+ "name": "INODES",
+ "integer": 2
+ }
+ ]
+ },
+ {
+ "name": "RPC.Type",
+ "enum_fields": [
+ {
+ "name": "UNKNOWN"
+ },
+ {
+ "name": "STAGE_UNSTAGE_VOLUME",
+ "integer": 1
+ },
+ {
+ "name": "GET_VOLUME_STATS",
+ "integer": 2
+ },
+ {
+ "name": "EXPAND_VOLUME",
+ "integer": 3
+ }
+ ]
+ }
+ ],
+ "messages": [
+ {
+ "name": "google.protobuf.FieldOptions",
+ "fields": [
+ {
+ "id": 1059,
+ "name": "csi_secret",
+ "type": "bool"
+ }
+ ]
+ },
+ {
+ "name": "GetPluginInfoRequest"
+ },
+ {
+ "name": "GetPluginInfoResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "name",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "vendor_version",
+ "type": "string"
+ }
+ ],
+ "maps": [
+ {
+ "key_type": "string",
+ "field": {
+ "id": 3,
+ "name": "manifest",
+ "type": "string"
+ }
+ }
+ ]
+ },
+ {
+ "name": "GetPluginCapabilitiesRequest"
+ },
+ {
+ "name": "GetPluginCapabilitiesResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "capabilities",
+ "type": "PluginCapability",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "PluginCapability",
+ "fields": [
+ {
+ "id": 1,
+ "name": "service",
+ "type": "Service"
+ },
+ {
+ "id": 2,
+ "name": "volume_expansion",
+ "type": "VolumeExpansion"
+ }
+ ],
+ "messages": [
+ {
+ "name": "Service",
+ "fields": [
+ {
+ "id": 1,
+ "name": "type",
+ "type": "Type"
+ }
+ ]
+ },
+ {
+ "name": "VolumeExpansion"
+ }
+ ]
+ },
+ {
+ "name": "ProbeRequest"
+ },
+ {
+ "name": "ProbeResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "ready",
+ "type": ".google.protobuf.BoolValue"
+ }
+ ]
+ },
+ {
+ "name": "CreateVolumeRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "name",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "capacity_range",
+ "type": "CapacityRange"
+ },
+ {
+ "id": 3,
+ "name": "volume_capabilities",
+ "type": "VolumeCapability",
+ "is_repeated": true
+ },
+ {
+ "id": 6,
+ "name": "volume_content_source",
+ "type": "VolumeContentSource"
+ },
+ {
+ "id": 7,
+ "name": "accessibility_requirements",
+ "type": "TopologyRequirement"
+ }
+ ],
+ "maps": [
+ {
+ "key_type": "string",
+ "field": {
+ "id": 4,
+ "name": "parameters",
+ "type": "string"
+ }
+ },
+ {
+ "key_type": "string",
+ "field": {
+ "id": 5,
+ "name": "secrets",
+ "type": "string",
+ "options": [
+ {
+ "name": "(csi_secret)",
+ "value": "true"
+ }
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "VolumeContentSource",
+ "fields": [
+ {
+ "id": 1,
+ "name": "snapshot",
+ "type": "SnapshotSource"
+ },
+ {
+ "id": 2,
+ "name": "volume",
+ "type": "VolumeSource"
+ }
+ ],
+ "messages": [
+ {
+ "name": "SnapshotSource",
+ "fields": [
+ {
+ "id": 1,
+ "name": "snapshot_id",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "VolumeSource",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volume_id",
+ "type": "string"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "CreateVolumeResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volume",
+ "type": "Volume"
+ }
+ ]
+ },
+ {
+ "name": "VolumeCapability",
+ "fields": [
+ {
+ "id": 1,
+ "name": "block",
+ "type": "BlockVolume"
+ },
+ {
+ "id": 2,
+ "name": "mount",
+ "type": "MountVolume"
+ },
+ {
+ "id": 3,
+ "name": "access_mode",
+ "type": "AccessMode"
+ }
+ ],
+ "messages": [
+ {
+ "name": "BlockVolume"
+ },
+ {
+ "name": "MountVolume",
+ "fields": [
+ {
+ "id": 1,
+ "name": "fs_type",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "mount_flags",
+ "type": "string",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "AccessMode",
+ "fields": [
+ {
+ "id": 1,
+ "name": "mode",
+ "type": "Mode"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "CapacityRange",
+ "fields": [
+ {
+ "id": 1,
+ "name": "required_bytes",
+ "type": "int64"
+ },
+ {
+ "id": 2,
+ "name": "limit_bytes",
+ "type": "int64"
+ }
+ ]
+ },
+ {
+ "name": "Volume",
+ "fields": [
+ {
+ "id": 1,
+ "name": "capacity_bytes",
+ "type": "int64"
+ },
+ {
+ "id": 2,
+ "name": "volume_id",
+ "type": "string"
+ },
+ {
+ "id": 4,
+ "name": "content_source",
+ "type": "VolumeContentSource"
+ },
+ {
+ "id": 5,
+ "name": "accessible_topology",
+ "type": "Topology",
+ "is_repeated": true
+ }
+ ],
+ "maps": [
+ {
+ "key_type": "string",
+ "field": {
+ "id": 3,
+ "name": "volume_context",
+ "type": "string"
+ }
+ }
+ ]
+ },
+ {
+ "name": "TopologyRequirement",
+ "fields": [
+ {
+ "id": 1,
+ "name": "requisite",
+ "type": "Topology",
+ "is_repeated": true
+ },
+ {
+ "id": 2,
+ "name": "preferred",
+ "type": "Topology",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "Topology",
+ "maps": [
+ {
+ "key_type": "string",
+ "field": {
+ "id": 1,
+ "name": "segments",
+ "type": "string"
+ }
+ }
+ ]
+ },
+ {
+ "name": "DeleteVolumeRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volume_id",
+ "type": "string"
+ }
+ ],
+ "maps": [
+ {
+ "key_type": "string",
+ "field": {
+ "id": 2,
+ "name": "secrets",
+ "type": "string",
+ "options": [
+ {
+ "name": "(csi_secret)",
+ "value": "true"
+ }
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "DeleteVolumeResponse"
+ },
+ {
+ "name": "ControllerPublishVolumeRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volume_id",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "node_id",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "volume_capability",
+ "type": "VolumeCapability"
+ },
+ {
+ "id": 4,
+ "name": "readonly",
+ "type": "bool"
+ }
+ ],
+ "maps": [
+ {
+ "key_type": "string",
+ "field": {
+ "id": 5,
+ "name": "secrets",
+ "type": "string",
+ "options": [
+ {
+ "name": "(csi_secret)",
+ "value": "true"
+ }
+ ]
+ }
+ },
+ {
+ "key_type": "string",
+ "field": {
+ "id": 6,
+ "name": "volume_context",
+ "type": "string"
+ }
+ }
+ ]
+ },
+ {
+ "name": "ControllerPublishVolumeResponse",
+ "maps": [
+ {
+ "key_type": "string",
+ "field": {
+ "id": 1,
+ "name": "publish_context",
+ "type": "string"
+ }
+ }
+ ]
+ },
+ {
+ "name": "ControllerUnpublishVolumeRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volume_id",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "node_id",
+ "type": "string"
+ }
+ ],
+ "maps": [
+ {
+ "key_type": "string",
+ "field": {
+ "id": 3,
+ "name": "secrets",
+ "type": "string",
+ "options": [
+ {
+ "name": "(csi_secret)",
+ "value": "true"
+ }
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "ControllerUnpublishVolumeResponse"
+ },
+ {
+ "name": "ValidateVolumeCapabilitiesRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volume_id",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "volume_capabilities",
+ "type": "VolumeCapability",
+ "is_repeated": true
+ }
+ ],
+ "maps": [
+ {
+ "key_type": "string",
+ "field": {
+ "id": 2,
+ "name": "volume_context",
+ "type": "string"
+ }
+ },
+ {
+ "key_type": "string",
+ "field": {
+ "id": 4,
+ "name": "parameters",
+ "type": "string"
+ }
+ },
+ {
+ "key_type": "string",
+ "field": {
+ "id": 5,
+ "name": "secrets",
+ "type": "string",
+ "options": [
+ {
+ "name": "(csi_secret)",
+ "value": "true"
+ }
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "ValidateVolumeCapabilitiesResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "confirmed",
+ "type": "Confirmed"
+ },
+ {
+ "id": 2,
+ "name": "message",
+ "type": "string"
+ }
+ ],
+ "messages": [
+ {
+ "name": "Confirmed",
+ "fields": [
+ {
+ "id": 2,
+ "name": "volume_capabilities",
+ "type": "VolumeCapability",
+ "is_repeated": true
+ }
+ ],
+ "maps": [
+ {
+ "key_type": "string",
+ "field": {
+ "id": 1,
+ "name": "volume_context",
+ "type": "string"
+ }
+ },
+ {
+ "key_type": "string",
+ "field": {
+ "id": 3,
+ "name": "parameters",
+ "type": "string"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "ListVolumesRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "max_entries",
+ "type": "int32"
+ },
+ {
+ "id": 2,
+ "name": "starting_token",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "ListVolumesResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "entries",
+ "type": "Entry",
+ "is_repeated": true
+ },
+ {
+ "id": 2,
+ "name": "next_token",
+ "type": "string"
+ }
+ ],
+ "messages": [
+ {
+ "name": "Entry",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volume",
+ "type": "Volume"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "GetCapacityRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volume_capabilities",
+ "type": "VolumeCapability",
+ "is_repeated": true
+ },
+ {
+ "id": 3,
+ "name": "accessible_topology",
+ "type": "Topology"
+ }
+ ],
+ "maps": [
+ {
+ "key_type": "string",
+ "field": {
+ "id": 2,
+ "name": "parameters",
+ "type": "string"
+ }
+ }
+ ]
+ },
+ {
+ "name": "GetCapacityResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "available_capacity",
+ "type": "int64"
+ }
+ ]
+ },
+ {
+ "name": "ControllerGetCapabilitiesRequest"
+ },
+ {
+ "name": "ControllerGetCapabilitiesResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "capabilities",
+ "type": "ControllerServiceCapability",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "ControllerServiceCapability",
+ "fields": [
+ {
+ "id": 1,
+ "name": "rpc",
+ "type": "RPC"
+ }
+ ],
+ "messages": [
+ {
+ "name": "RPC",
+ "fields": [
+ {
+ "id": 1,
+ "name": "type",
+ "type": "Type"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "CreateSnapshotRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "source_volume_id",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "name",
+ "type": "string"
+ }
+ ],
+ "maps": [
+ {
+ "key_type": "string",
+ "field": {
+ "id": 3,
+ "name": "secrets",
+ "type": "string",
+ "options": [
+ {
+ "name": "(csi_secret)",
+ "value": "true"
+ }
+ ]
+ }
+ },
+ {
+ "key_type": "string",
+ "field": {
+ "id": 4,
+ "name": "parameters",
+ "type": "string"
+ }
+ }
+ ]
+ },
+ {
+ "name": "CreateSnapshotResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "snapshot",
+ "type": "Snapshot"
+ }
+ ]
+ },
+ {
+ "name": "Snapshot",
+ "fields": [
+ {
+ "id": 1,
+ "name": "size_bytes",
+ "type": "int64"
+ },
+ {
+ "id": 2,
+ "name": "snapshot_id",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "source_volume_id",
+ "type": "string"
+ },
+ {
+ "id": 4,
+ "name": "creation_time",
+ "type": ".google.protobuf.Timestamp"
+ },
+ {
+ "id": 5,
+ "name": "ready_to_use",
+ "type": "bool"
+ }
+ ]
+ },
+ {
+ "name": "DeleteSnapshotRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "snapshot_id",
+ "type": "string"
+ }
+ ],
+ "maps": [
+ {
+ "key_type": "string",
+ "field": {
+ "id": 2,
+ "name": "secrets",
+ "type": "string",
+ "options": [
+ {
+ "name": "(csi_secret)",
+ "value": "true"
+ }
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "DeleteSnapshotResponse"
+ },
+ {
+ "name": "ListSnapshotsRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "max_entries",
+ "type": "int32"
+ },
+ {
+ "id": 2,
+ "name": "starting_token",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "source_volume_id",
+ "type": "string"
+ },
+ {
+ "id": 4,
+ "name": "snapshot_id",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "ListSnapshotsResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "entries",
+ "type": "Entry",
+ "is_repeated": true
+ },
+ {
+ "id": 2,
+ "name": "next_token",
+ "type": "string"
+ }
+ ],
+ "messages": [
+ {
+ "name": "Entry",
+ "fields": [
+ {
+ "id": 1,
+ "name": "snapshot",
+ "type": "Snapshot"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "ControllerExpandVolumeRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volume_id",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "capacity_range",
+ "type": "CapacityRange"
+ }
+ ],
+ "maps": [
+ {
+ "key_type": "string",
+ "field": {
+ "id": 3,
+ "name": "secrets",
+ "type": "string",
+ "options": [
+ {
+ "name": "(csi_secret)",
+ "value": "true"
+ }
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "ControllerExpandVolumeResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "capacity_bytes",
+ "type": "int64"
+ },
+ {
+ "id": 2,
+ "name": "node_expansion_required",
+ "type": "bool"
+ }
+ ]
+ },
+ {
+ "name": "NodeStageVolumeRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volume_id",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "staging_target_path",
+ "type": "string"
+ },
+ {
+ "id": 4,
+ "name": "volume_capability",
+ "type": "VolumeCapability"
+ }
+ ],
+ "maps": [
+ {
+ "key_type": "string",
+ "field": {
+ "id": 2,
+ "name": "publish_context",
+ "type": "string"
+ }
+ },
+ {
+ "key_type": "string",
+ "field": {
+ "id": 5,
+ "name": "secrets",
+ "type": "string",
+ "options": [
+ {
+ "name": "(csi_secret)",
+ "value": "true"
+ }
+ ]
+ }
+ },
+ {
+ "key_type": "string",
+ "field": {
+ "id": 6,
+ "name": "volume_context",
+ "type": "string"
+ }
+ }
+ ]
+ },
+ {
+ "name": "NodeStageVolumeResponse"
+ },
+ {
+ "name": "NodeUnstageVolumeRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volume_id",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "staging_target_path",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "NodeUnstageVolumeResponse"
+ },
+ {
+ "name": "NodePublishVolumeRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volume_id",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "staging_target_path",
+ "type": "string"
+ },
+ {
+ "id": 4,
+ "name": "target_path",
+ "type": "string"
+ },
+ {
+ "id": 5,
+ "name": "volume_capability",
+ "type": "VolumeCapability"
+ },
+ {
+ "id": 6,
+ "name": "readonly",
+ "type": "bool"
+ }
+ ],
+ "maps": [
+ {
+ "key_type": "string",
+ "field": {
+ "id": 2,
+ "name": "publish_context",
+ "type": "string"
+ }
+ },
+ {
+ "key_type": "string",
+ "field": {
+ "id": 7,
+ "name": "secrets",
+ "type": "string",
+ "options": [
+ {
+ "name": "(csi_secret)",
+ "value": "true"
+ }
+ ]
+ }
+ },
+ {
+ "key_type": "string",
+ "field": {
+ "id": 8,
+ "name": "volume_context",
+ "type": "string"
+ }
+ }
+ ]
+ },
+ {
+ "name": "NodePublishVolumeResponse"
+ },
+ {
+ "name": "NodeUnpublishVolumeRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volume_id",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "target_path",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "NodeUnpublishVolumeResponse"
+ },
+ {
+ "name": "NodeGetVolumeStatsRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volume_id",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "volume_path",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "name": "NodeGetVolumeStatsResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "usage",
+ "type": "VolumeUsage",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "VolumeUsage",
+ "fields": [
+ {
+ "id": 1,
+ "name": "available",
+ "type": "int64"
+ },
+ {
+ "id": 2,
+ "name": "total",
+ "type": "int64"
+ },
+ {
+ "id": 3,
+ "name": "used",
+ "type": "int64"
+ },
+ {
+ "id": 4,
+ "name": "unit",
+ "type": "Unit"
+ }
+ ]
+ },
+ {
+ "name": "NodeGetCapabilitiesRequest"
+ },
+ {
+ "name": "NodeGetCapabilitiesResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "capabilities",
+ "type": "NodeServiceCapability",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "NodeServiceCapability",
+ "fields": [
+ {
+ "id": 1,
+ "name": "rpc",
+ "type": "RPC"
+ }
+ ],
+ "messages": [
+ {
+ "name": "RPC",
+ "fields": [
+ {
+ "id": 1,
+ "name": "type",
+ "type": "Type"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "NodeGetInfoRequest"
+ },
+ {
+ "name": "NodeGetInfoResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "node_id",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "max_volumes_per_node",
+ "type": "int64"
+ },
+ {
+ "id": 3,
+ "name": "accessible_topology",
+ "type": "Topology"
+ }
+ ]
+ },
+ {
+ "name": "NodeExpandVolumeRequest",
+ "fields": [
+ {
+ "id": 1,
+ "name": "volume_id",
+ "type": "string"
+ },
+ {
+ "id": 2,
+ "name": "volume_path",
+ "type": "string"
+ },
+ {
+ "id": 3,
+ "name": "capacity_range",
+ "type": "CapacityRange"
+ }
+ ]
+ },
+ {
+ "name": "NodeExpandVolumeResponse",
+ "fields": [
+ {
+ "id": 1,
+ "name": "capacity_bytes",
+ "type": "int64"
+ }
+ ]
+ }
+ ],
+ "services": [
+ {
+ "name": "Identity",
+ "rpcs": [
+ {
+ "name": "GetPluginInfo",
+ "in_type": "GetPluginInfoRequest",
+ "out_type": "GetPluginInfoResponse"
+ },
+ {
+ "name": "GetPluginCapabilities",
+ "in_type": "GetPluginCapabilitiesRequest",
+ "out_type": "GetPluginCapabilitiesResponse"
+ },
+ {
+ "name": "Probe",
+ "in_type": "ProbeRequest",
+ "out_type": "ProbeResponse"
+ }
+ ]
+ },
+ {
+ "name": "Controller",
+ "rpcs": [
+ {
+ "name": "CreateVolume",
+ "in_type": "CreateVolumeRequest",
+ "out_type": "CreateVolumeResponse"
+ },
+ {
+ "name": "DeleteVolume",
+ "in_type": "DeleteVolumeRequest",
+ "out_type": "DeleteVolumeResponse"
+ },
+ {
+ "name": "ControllerPublishVolume",
+ "in_type": "ControllerPublishVolumeRequest",
+ "out_type": "ControllerPublishVolumeResponse"
+ },
+ {
+ "name": "ControllerUnpublishVolume",
+ "in_type": "ControllerUnpublishVolumeRequest",
+ "out_type": "ControllerUnpublishVolumeResponse"
+ },
+ {
+ "name": "ValidateVolumeCapabilities",
+ "in_type": "ValidateVolumeCapabilitiesRequest",
+ "out_type": "ValidateVolumeCapabilitiesResponse"
+ },
+ {
+ "name": "ListVolumes",
+ "in_type": "ListVolumesRequest",
+ "out_type": "ListVolumesResponse"
+ },
+ {
+ "name": "GetCapacity",
+ "in_type": "GetCapacityRequest",
+ "out_type": "GetCapacityResponse"
+ },
+ {
+ "name": "ControllerGetCapabilities",
+ "in_type": "ControllerGetCapabilitiesRequest",
+ "out_type": "ControllerGetCapabilitiesResponse"
+ },
+ {
+ "name": "CreateSnapshot",
+ "in_type": "CreateSnapshotRequest",
+ "out_type": "CreateSnapshotResponse"
+ },
+ {
+ "name": "DeleteSnapshot",
+ "in_type": "DeleteSnapshotRequest",
+ "out_type": "DeleteSnapshotResponse"
+ },
+ {
+ "name": "ListSnapshots",
+ "in_type": "ListSnapshotsRequest",
+ "out_type": "ListSnapshotsResponse"
+ },
+ {
+ "name": "ControllerExpandVolume",
+ "in_type": "ControllerExpandVolumeRequest",
+ "out_type": "ControllerExpandVolumeResponse"
+ }
+ ]
+ },
+ {
+ "name": "Node",
+ "rpcs": [
+ {
+ "name": "NodeStageVolume",
+ "in_type": "NodeStageVolumeRequest",
+ "out_type": "NodeStageVolumeResponse"
+ },
+ {
+ "name": "NodeUnstageVolume",
+ "in_type": "NodeUnstageVolumeRequest",
+ "out_type": "NodeUnstageVolumeResponse"
+ },
+ {
+ "name": "NodePublishVolume",
+ "in_type": "NodePublishVolumeRequest",
+ "out_type": "NodePublishVolumeResponse"
+ },
+ {
+ "name": "NodeUnpublishVolume",
+ "in_type": "NodeUnpublishVolumeRequest",
+ "out_type": "NodeUnpublishVolumeResponse"
+ },
+ {
+ "name": "NodeGetVolumeStats",
+ "in_type": "NodeGetVolumeStatsRequest",
+ "out_type": "NodeGetVolumeStatsResponse"
+ },
+ {
+ "name": "NodeExpandVolume",
+ "in_type": "NodeExpandVolumeRequest",
+ "out_type": "NodeExpandVolumeResponse"
+ },
+ {
+ "name": "NodeGetCapabilities",
+ "in_type": "NodeGetCapabilitiesRequest",
+ "out_type": "NodeGetCapabilitiesResponse"
+ },
+ {
+ "name": "NodeGetInfo",
+ "in_type": "NodeGetInfoRequest",
+ "out_type": "NodeGetInfoResponse"
+ }
+ ]
+ }
+ ],
+ "imports": [
+ {
+ "path": "google/protobuf/descriptor.proto"
+ },
+ {
+ "path": "google/protobuf/timestamp.proto"
+ },
+ {
+ "path": "google/protobuf/wrappers.proto"
+ }
+ ],
+ "package": {
+ "name": "csi.v1"
+ },
+ "options": [
+ {
+ "name": "go_package",
+ "value": "csi"
+ }
+ ]
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/hadoop-ozone/datanode/pom.xml b/hadoop-ozone/datanode/pom.xml
index 6e1bf11..75eaa8c 100644
--- a/hadoop-ozone/datanode/pom.xml
+++ b/hadoop-ozone/datanode/pom.xml
@@ -29,7 +29,7 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
+ <artifactId>hadoop-hdds-hadoop-dependency-server</artifactId>
<scope>compile</scope>
<exclusions>
<exclusion>
diff --git a/hadoop-ozone/dev-support/checks/author.sh b/hadoop-ozone/dev-support/checks/author.sh
index 92903f9..a4e7515 100755
--- a/hadoop-ozone/dev-support/checks/author.sh
+++ b/hadoop-ozone/dev-support/checks/author.sh
@@ -20,11 +20,7 @@
mkdir -p "$REPORT_DIR"
REPORT_FILE="$REPORT_DIR/summary.txt"
-#hide this string to not confuse yetus
-AUTHOR="uthor"
-AUTHOR="@a${AUTHOR}"
-
-grep -r --include="*.java" "$AUTHOR" . | tee "$REPORT_FILE"
+grep -r --include="*.java" "@author" . | tee "$REPORT_FILE"
wc -l "$REPORT_FILE" | awk '{print $1}'> "$REPORT_DIR/failures"
diff --git a/hadoop-ozone/dev-support/checks/checkstyle.sh b/hadoop-ozone/dev-support/checks/checkstyle.sh
index bdae1d6..a433391 100755
--- a/hadoop-ozone/dev-support/checks/checkstyle.sh
+++ b/hadoop-ozone/dev-support/checks/checkstyle.sh
@@ -21,7 +21,18 @@
mkdir -p "$REPORT_DIR"
REPORT_FILE="$REPORT_DIR/summary.txt"
-mvn -B -fn checkstyle:check
+MAVEN_OPTIONS='-B -fae -Dskip.yarn -Dskip.installyarn -Dcheckstyle.failOnViolation=false'
+
+declare -i rc
+mvn ${MAVEN_OPTIONS} checkstyle:check > "${REPORT_DIR}/output.log"
+rc=$?
+if [[ ${rc} -ne 0 ]]; then
+ mvn ${MAVEN_OPTIONS} clean test-compile checkstyle:check
+ rc=$?
+ mkdir -p "$REPORT_DIR" # removed by mvn clean
+else
+ cat "${REPORT_DIR}/output.log"
+fi
#Print out the exact violations with parsing XML results with sed
find "." -name checkstyle-errors.xml -print0 \
@@ -41,3 +52,4 @@
if [[ -s "${REPORT_FILE}" ]]; then
exit 1
fi
+exit ${rc}
diff --git a/hadoop-ozone/dev-support/intellij/ozone-site.xml b/hadoop-ozone/dev-support/intellij/ozone-site.xml
index d2b9b64..3fde850 100644
--- a/hadoop-ozone/dev-support/intellij/ozone-site.xml
+++ b/hadoop-ozone/dev-support/intellij/ozone-site.xml
@@ -24,6 +24,14 @@
<value>localhost</value>
</property>
<property>
+ <name>ozone.csi.owner</name>
+ <value>hadoop</value>
+ </property>
+ <property>
+ <name>ozone.csi.socket</name>
+ <value>/tmp/csi.sock</value>
+ </property>
+ <property>
<name>ozone.scm.client.address</name>
<value>localhost</value>
</property>
diff --git a/hadoop-ozone/dev-support/intellij/runConfigurations/CsiServer.xml b/hadoop-ozone/dev-support/intellij/runConfigurations/CsiServer.xml
new file mode 100644
index 0000000..d3bb7ab
--- /dev/null
+++ b/hadoop-ozone/dev-support/intellij/runConfigurations/CsiServer.xml
@@ -0,0 +1,33 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<component name="ProjectRunConfigurationManager">
+ <configuration default="false" name="CsiServer" type="Application" factoryName="Application" nameIsGenerated="true">
+ <option name="MAIN_CLASS_NAME" value="org.apache.hadoop.ozone.csi.CsiServer" />
+ <module name="hadoop-ozone-csi" />
+ <option name="PROGRAM_PARAMETERS" value="-conf=hadoop-ozone/dev-support/intellij/ozone-site.xml" />
+ <option name="VM_PARAMETERS" value="-Dlog4j.configuration=file:hadoop-ozone/dev-support/intellij/log4j.properties" />
+ <extension name="coverage">
+ <pattern>
+ <option name="PATTERN" value="org.apache.hadoop.ozone.csi.*" />
+ <option name="ENABLED" value="true" />
+ </pattern>
+ </extension>
+ <method v="2">
+ <option name="Make" enabled="true" />
+ </method>
+ </configuration>
+</component>
\ No newline at end of file
diff --git a/hadoop-ozone/dev-support/intellij/runConfigurations/OzoneShell.xml b/hadoop-ozone/dev-support/intellij/runConfigurations/OzoneShell.xml
index 2d0bf80..32a0a4c 100644
--- a/hadoop-ozone/dev-support/intellij/runConfigurations/OzoneShell.xml
+++ b/hadoop-ozone/dev-support/intellij/runConfigurations/OzoneShell.xml
@@ -16,8 +16,8 @@
-->
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="OzoneShell" type="Application" factoryName="Application">
- <option name="MAIN_CLASS_NAME" value="org.apache.hadoop.ozone.web.ozShell.OzoneShell" />
- <module name="hadoop-ozone-ozone-manager" />
+ <option name="MAIN_CLASS_NAME" value="org.apache.hadoop.ozone.shell.OzoneShell" />
+ <module name="hadoop-ozone-tools" />
<option name="PROGRAM_PARAMETERS" value="-conf=hadoop-ozone/dev-support/intellij/ozone-site.xml volume create /vol1" />
<option name="VM_PARAMETERS" value="-Dlog4j.configuration=file:hadoop-ozone/dev-support/intellij/log4j.properties" />
<extension name="coverage">
@@ -30,4 +30,4 @@
<option name="Make" enabled="true" />
</method>
</configuration>
-</component>
\ No newline at end of file
+</component>
diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml
index 803abf4..896831b 100644
--- a/hadoop-ozone/dist/pom.xml
+++ b/hadoop-ozone/dist/pom.xml
@@ -23,12 +23,12 @@
</parent>
<artifactId>hadoop-ozone-dist</artifactId>
<name>Apache Hadoop Ozone Distribution</name>
- <packaging>pom</packaging>
+ <packaging>jar</packaging>
<version>0.6.0-SNAPSHOT</version>
<properties>
<file.encoding>UTF-8</file.encoding>
<downloadSources>true</downloadSources>
- <docker.ozone-runner.version>20191107-1</docker.ozone-runner.version>
+ <docker.ozone-runner.version>20200420-1</docker.ozone-runner.version>
</properties>
<build>
@@ -147,8 +147,7 @@
<goal>copy-dependencies</goal>
</goals>
<configuration>
- <outputDirectory>target/ozone-${ozone.version}/share/ozone/lib
- </outputDirectory>
+ <outputDirectory>target/ozone-${ozone.version}/share/ozone/lib</outputDirectory>
<includeScope>runtime</includeScope>
</configuration>
</execution>
@@ -165,12 +164,7 @@
<artifactItem>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
- <version>3.5.1</version>
- </artifactItem>
- <artifactItem>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>26.0-android</version>
+ <version>${protobuf-compile.version}</version>
</artifactItem>
</artifactItems>
</configuration>
@@ -179,7 +173,6 @@
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
- <version>3.1.0</version>
<executions>
<execution>
<id>copy-compose-files</id>
@@ -434,7 +427,6 @@
<plugin>
<groupId>io.fabric8</groupId>
<artifactId>docker-maven-plugin</artifactId>
- <version>0.29.0</version>
<executions>
<execution>
<goals>
@@ -466,7 +458,6 @@
<plugin>
<groupId>io.fabric8</groupId>
<artifactId>docker-maven-plugin</artifactId>
- <version>0.29.0</version>
<executions>
<execution>
<goals>
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-csi/.env b/hadoop-ozone/dist/src/main/compose/ozone-csi/.env
new file mode 100644
index 0000000..96ab163
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozone-csi/.env
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HDDS_VERSION=${hdds.version}
+OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-compose.yaml
new file mode 100644
index 0000000..e175dfa
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-compose.yaml
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3.4"
+
+services:
+ datanode:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ volumes:
+ - ../..:/opt/hadoop
+ env_file:
+ - docker-config
+ ports:
+ - 9864
+ - 9882
+ command: ["ozone","datanode"]
+ om:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ volumes:
+ - ../..:/opt/hadoop
+ env_file:
+ - docker-config
+ environment:
+ ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+ ports:
+ - 9874:9874
+ - 9862:9862
+ command: ["ozone","om"]
+ scm:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ volumes:
+ - ../..:/opt/hadoop
+ env_file:
+ - docker-config
+ ports:
+ - 9876:9876
+ environment:
+ ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+ command: ["ozone","scm"]
+ csi:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ volumes:
+ - ../..:/opt/hadoop
+ env_file:
+ - docker-config
+ command: ["ozone","csi"]
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config
new file mode 100644
index 0000000..adc6646
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OZONE-SITE.XML_ozone.csi.owner=hadoop
+OZONE-SITE.XML_ozone.csi.socket=/tmp/csi.sock
+
+OZONE-SITE.XML_ozone.om.address=om
+OZONE-SITE.XML_ozone.om.http-address=om:9874
+OZONE-SITE.XML_ozone.scm.container.size=1GB
+OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
+OZONE-SITE.XML_ozone.scm.client.address=scm
+OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
+
+no_proxy=om,scm,csi,s3g,kdc,localhost,127.0.0.1
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-csi/test.sh
old mode 100644
new mode 100755
similarity index 68%
copy from hadoop-ozone/dist/src/main/compose/ozone-om-ha/test.sh
copy to hadoop-ozone/dist/src/main/compose/ozone-csi/test.sh
index 3146ae7..2a6e5f1
--- a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone-csi/test.sh
@@ -15,25 +15,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-startOMs(){
- docker-compose exec -T om1 /opt/startOM.sh
- docker-compose exec -T om2 /opt/startOM.sh
- docker-compose exec -T om3 /opt/startOM.sh
-}
-
-COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+COMPOSE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
export COMPOSE_DIR
+export SECURITY_ENABLED=false
+export OZONE_REPLICATION_FACTOR=3
+
# shellcheck source=/dev/null
source "$COMPOSE_DIR/../testlib.sh"
start_docker_env
-# Start OMs separately. In this test, the OMs will be stopped and restarted multiple times.
-# So we do not want the container to be tied to the OM process.
-startOMs
-
-execute_robot_test scm omha/testOMHA.robot
+execute_robot_test csi csi.robot
stop_docker_env
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-compose.yaml
index 43e1682..77939e9 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-compose.yaml
@@ -42,6 +42,7 @@
- ../..:/opt/hadoop
ports:
- 9874:9874
+ - 9862:9862
environment:
ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
env_file:
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config b/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config
index a356c92..46e7527 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config
+++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config
@@ -30,7 +30,7 @@
HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
-CORE-SITE.xml_fs.defaultFS=o3fs://bucket1.vol1/
+CORE-SITE.xml_fs.defaultFS=o3fs://bucket1.volume1/
MAPRED-SITE.XML_mapreduce.framework.name=yarn
MAPRED-SITE.XML_yarn.app.mapreduce.am.env=HADOOP_MAPRED_HOME=$HADOOP_HOME
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-compose.yaml
index 85f75b0..37afe2f 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-compose.yaml
@@ -33,6 +33,7 @@
- ../../..:/opt/hadoop
ports:
- 9874:9874
+ - 9862:9862
environment:
WAITFOR: scm:9876
ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-compose.yaml
index 9a30c8e..05464d4 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-compose.yaml
@@ -33,6 +33,7 @@
- ../../..:/opt/hadoop
ports:
- 9874:9874
+ - 9862:9862
environment:
WAITFOR: scm:9876
ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/test.sh
index 457dd11..362ff98 100755
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/test.sh
@@ -19,6 +19,7 @@
export COMPOSE_DIR
# shellcheck source=/dev/null
+source "$COMPOSE_DIR/.env"
source "$COMPOSE_DIR/../../testlib.sh"
start_docker_env
@@ -35,9 +36,10 @@
execute_robot_test rm ozonefs/hadoopo3fs.robot
-execute_robot_test rm -v hadoop.version:3.1.2 mapreduce.robot
-
+execute_robot_test rm -v "hadoop.version:${HADOOP_VERSION}" mapreduce.robot
stop_docker_env
generate_report
+
+cleanup_docker_images "${HADOOP_IMAGE}:${HADOOP_VERSION}"
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-compose.yaml
index 3462a91..aee8910 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-compose.yaml
@@ -33,6 +33,7 @@
- ../../..:/opt/hadoop
ports:
- 9874:9874
+ - 9862:9862
environment:
WAITFOR: scm:9876
ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/test.sh
index 6ffb945..719ad4c 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/test.sh
@@ -19,7 +19,7 @@
export COMPOSE_DIR
export SECURITY_ENABLED=false
-export OM_HA_PARAM="--om-service-id=id1"
+export OM_SERVICE_ID="id1"
# shellcheck source=/dev/null
source "$COMPOSE_DIR/../testlib.sh"
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config
index feafda9..d589210 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config
@@ -15,7 +15,7 @@
# limitations under the License.
CORE-SITE.XML_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
-CORE-SITE.XML_fs.defaultFS=o3fs://bucket.volume.omservice
+CORE-SITE.XML_fs.defaultFS=o3fs://bucket1.volume1.omservice
OZONE-SITE.XML_ozone.om.service.ids=omservice
OZONE-SITE.XML_ozone.om.nodes.omservice=om1,om2,om3
OZONE-SITE.XML_ozone.om.address.omservice.om1=om1
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/test_disabled.sh
similarity index 97%
rename from hadoop-ozone/dist/src/main/compose/ozone-om-ha/test.sh
rename to hadoop-ozone/dist/src/main/compose/ozone-om-ha/test_disabled.sh
index 3146ae7..eb9c27b 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/test_disabled.sh
@@ -23,6 +23,7 @@
COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
export COMPOSE_DIR
+export OM_SERVICE_ID="omservice"
# shellcheck source=/dev/null
source "$COMPOSE_DIR/../testlib.sh"
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-compose.yaml
index ccd131c..7086c3f 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-compose.yaml
@@ -107,6 +107,7 @@
- ../..:/opt/hadoop
ports:
- 9874:9874
+ - 9862:9862
environment:
ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
env_file:
@@ -126,7 +127,7 @@
- ./docker-config
environment:
ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
- OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}"
+ OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-6}"
command: ["/opt/hadoop/bin/ozone","scm"]
networks:
net:
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config
index 3556f8f..e0c54ea 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config
@@ -16,7 +16,8 @@
OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
-OZONE-SITE.XML_ozone.scm.container.size=1GB
+OZONE-SITE.XML_ozone.scm.container.size=256MB
+OZONE-SITE.XML_ozone.ozone.scm.block.size=64MB
OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-topology/hdds-3084.sh b/hadoop-ozone/dist/src/main/compose/ozone-topology/hdds-3084.sh
deleted file mode 100755
index d4efa4f..0000000
--- a/hadoop-ozone/dist/src/main/compose/ozone-topology/hdds-3084.sh
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-export COMPOSE_DIR
-
-# shellcheck source=/dev/null
-source "$COMPOSE_DIR/../testlib.sh"
-
-start_docker_env 4
-
-#Due to the limitation of the current auditparser test, it should be the
-#first test in a clean cluster.
-
-#Disabling for now, audit parser tool during parse getting exception.
-#execute_robot_test om auditparser
-
-execute_robot_test scm basic/basic.robot
-
-execute_robot_test scm topology/cli.robot
-
-# Ensure data can be read even when a full rack
-# is stopped.
-execute_robot_test scm topology/loaddata.robot
-
-stop_containers datanode_1 datanode_2 datanode_3
-
-execute_robot_test scm topology/readdata.robot
-
-start_containers datanode_1 datanode_2 datanode_3
-
-wait_for_port datanode_1 9858 60
-wait_for_port datanode_2 9858 60
-wait_for_port datanode_3 9858 60
-
-stop_containers datanode_4 datanode_5 datanode_6
-
-execute_robot_test scm topology/readdata.robot
-
-stop_docker_env
-
-generate_report
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-topology/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-topology/test.sh
index a964634..d4efa4f 100755
--- a/hadoop-ozone/dist/src/main/compose/ozone-topology/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone-topology/test.sh
@@ -33,6 +33,24 @@
execute_robot_test scm topology/cli.robot
+# Ensure data can be read even when a full rack
+# is stopped.
+execute_robot_test scm topology/loaddata.robot
+
+stop_containers datanode_1 datanode_2 datanode_3
+
+execute_robot_test scm topology/readdata.robot
+
+start_containers datanode_1 datanode_2 datanode_3
+
+wait_for_port datanode_1 9858 60
+wait_for_port datanode_2 9858 60
+wait_for_port datanode_3 9858 60
+
+stop_containers datanode_4 datanode_5 datanode_6
+
+execute_robot_test scm topology/readdata.robot
+
stop_docker_env
generate_report
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml
index 0debaca..869aa05 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml
@@ -45,6 +45,7 @@
<<: *replication
ports:
- 9874:9874
+ - 9862:9862
command: ["ozone","om"]
scm:
<<: *common-config
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test.sh b/hadoop-ozone/dist/src/main/compose/ozone/test.sh
index 735a40b..b77bd55 100755
--- a/hadoop-ozone/dist/src/main/compose/ozone/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone/test.sh
@@ -38,6 +38,8 @@
execute_robot_test scm ozonefs/ozonefs.robot
+execute_robot_test scm security/ozone-secure-token.robot
+
execute_robot_test scm s3
execute_robot_test scm recon
diff --git a/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-compose.yaml
index 703329f..c34c520 100644
--- a/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-compose.yaml
@@ -31,6 +31,7 @@
- ../..:/opt/hadoop
ports:
- 9874:9874
+ - 9862:9862
environment:
ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
env_file:
diff --git a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml
index 25ed6f6..2a1b5a0 100644
--- a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml
@@ -38,6 +38,7 @@
- ../..:/opt/hadoop
ports:
- 9874:9874
+ - 9862:9862
environment:
ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
env_file:
diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-compose.yaml
index e19dd23..826ca5c 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-compose.yaml
@@ -38,6 +38,7 @@
- ../..:/opt/hadoop
ports:
- 9874:9874
+ - 9862:9862
env_file:
- ./docker-config
scm:
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/README.md b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/README.md
index 1426270..63a09a9 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/README.md
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/README.md
@@ -31,13 +31,13 @@
kinit -kt /etc/security/keytabs/testuser.keytab testuser/om@EXAMPLE.COM
-ozone sh volume create /vol1
+ozone sh volume create /volume1
-ozone sh bucket create /vol1/bucket1
+ozone sh bucket create /volume1/bucket1
-ozone sh key put /vol1/bucket1/key1 LICENSE.txt
+ozone sh key put /volume1/bucket1/key1 LICENSE.txt
-ozone fs -ls o3fs://bucket1.vol1/
+ozone fs -ls o3fs://bucket1.volume1/
```
## Yarn Resource Manager Setup
@@ -57,7 +57,7 @@
### WordCount
```
-yarn jar $HADOOP_MAPRED_HOME/hadoop-mapreduce-examples-*.jar wordcount o3fs://bucket1.vol1/key1 o3fs://bucket1.vol1/key1.count
+yarn jar $HADOOP_MAPRED_HOME/hadoop-mapreduce-examples-*.jar wordcount o3fs://bucket1.volume1/key1 o3fs://bucket1.volume1/key1.count
hadoop fs -cat /key1.count/part-r-00000
```
@@ -69,5 +69,5 @@
### RandomWrite
```
-yarn jar $HADOOP_MAPRED_HOME/hadoop-mapreduce-examples-*.jar randomwriter -Dtest.randomwrite.total_bytes=10000000 o3fs://bucket1.vol1/randomwrite.out
+yarn jar $HADOOP_MAPRED_HOME/hadoop-mapreduce-examples-*.jar randomwriter -Dtest.randomwrite.total_bytes=10000000 o3fs://bucket1.volume1/randomwrite.out
```
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml
index 5e58eba..1079682 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml
@@ -57,6 +57,7 @@
- ../..:/opt/hadoop
ports:
- 9874:9874
+ - 9862:9862
environment:
ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
KERBEROS_KEYTABS: om HTTP testuser
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config
index f0b7f5c..4497bb0 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config
@@ -30,15 +30,18 @@
OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
-OZONE-SITE.XML_ozone.s3g.keytab.file=/etc/security/keytabs/HTTP.keytab
-OZONE-SITE.XML_ozone.s3g.authentication.kerberos.principal=HTTP/s3g@EXAMPLE.COM
OZONE-SITE.XML_ozone.administrators=*
OZONE-SITE.XML_ozone.security.enabled=true
-OZONE-SITE.XML_hdds.scm.http.kerberos.principal=HTTP/scm@EXAMPLE.COM
-OZONE-SITE.XML_hdds.scm.http.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-OZONE-SITE.XML_ozone.om.http.kerberos.principal=HTTP/om@EXAMPLE.COM
-OZONE-SITE.XML_ozone.om.http.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal=HTTP/scm@EXAMPLE.COM
+OZONE-SITE.XML_hdds.scm.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.om.http.auth.kerberos.principal=HTTP/om@EXAMPLE.COM
+OZONE-SITE.XML_ozone.om.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
+OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.principal=HTTP/s3g@EXAMPLE.COM
+
HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/_HOST@EXAMPLE.COM
HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
@@ -64,7 +67,7 @@
HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
CORE-SITE.XML_fs.AbstractFileSystem.o3fs.impl=org.apache.hadoop.fs.ozone.OzFs
-CORE-SITE.XML_fs.defaultFS=o3fs://bucket1.vol1/
+CORE-SITE.XML_fs.defaultFS=o3fs://bucket1.volume1/
MAPRED-SITE.XML_mapreduce.framework.name=yarn
MAPRED-SITE.XML_yarn.app.mapreduce.am.env=HADOOP_MAPRED_HOME=$HADOOP_HOME
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/.env b/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/.env
new file mode 100644
index 0000000..37227ac
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/.env
@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HDDS_VERSION=${hdds.version}
+HADOOP_VERSION=3
+OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-compose.yaml
new file mode 100644
index 0000000..ba0bd19
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-compose.yaml
@@ -0,0 +1,201 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+ kdc:
+ build:
+ context: ../common/docker-image/docker-krb5
+ dockerfile: Dockerfile-krb5
+ image: ozone-insecure-krb5
+ hostname: kdc
+ volumes:
+ - ../..:/opt/hadoop
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.100
+ kms:
+ image: apache/hadoop:${HADOOP_VERSION}
+ ports:
+ - 9600:9600
+ env_file:
+ - ./docker-config
+ command: ["hadoop", "kms"]
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.101
+ datanode1:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9864:9999
+ command: ["/opt/hadoop/bin/ozone","datanode"]
+ env_file:
+ - docker-config
+ environment:
+ KERBEROS_KEYTABS: dn HTTP
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.102
+ datanode2:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9866:9999
+ command: ["/opt/hadoop/bin/ozone","datanode"]
+ env_file:
+ - docker-config
+ environment:
+ KERBEROS_KEYTABS: dn HTTP
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.103
+ datanode3:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9868:9999
+ command: ["/opt/hadoop/bin/ozone","datanode"]
+ env_file:
+ - docker-config
+ environment:
+ KERBEROS_KEYTABS: dn HTTP
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.104
+ om1:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ hostname: om1
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9880:9874
+ - 9890:9872
+ #- 18001:18001
+ environment:
+ ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+ KERBEROS_KEYTABS: om HTTP
+ #HADOOP_OPTS: "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:18001"
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/ozone","om"]
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.111
+ om2:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ hostname: om2
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9882:9874
+ - 9892:9872
+ #- 18002:18002
+ environment:
+ ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+ KERBEROS_KEYTABS: om HTTP
+ #HADOOP_OPTS: "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:18002"
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/ozone","om"]
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.112
+ om3:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ hostname: om3
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9884:9874
+ - 9894:9872
+ #- 18003:18003
+ environment:
+ ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+ KERBEROS_KEYTABS: om HTTP
+ #HADOOP_OPTS: "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:18003"
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/ozone","om"]
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.113
+ s3g:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ hostname: s3g
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9878:9878
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/ozone","s3g"]
+ environment:
+ KERBEROS_KEYTABS: s3g HTTP testuser
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.114
+ recon:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ hostname: recon
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9888:9888
+ #- 18000:18000
+ env_file:
+ - ./docker-config
+ environment:
+ KERBEROS_KEYTABS: recon HTTP
+ #HADOOP_OPTS: "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:18000"
+ command: ["/opt/hadoop/bin/ozone","recon"]
+ extra_hosts:
+ - "om1: 172.25.0.111"
+ - "om2: 172.25.0.112"
+ - "om3: 172.25.0.113"
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.115
+ scm:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ hostname: scm
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9876:9876
+ env_file:
+ - docker-config
+ environment:
+ KERBEROS_KEYTABS: scm HTTP testuser testuser2
+ ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+ OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-3}"
+ command: ["/opt/hadoop/bin/ozone","scm"]
+ extra_hosts:
+ - "om1: 172.25.0.111"
+ - "om2: 172.25.0.112"
+ - "om3: 172.25.0.113"
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.116
+networks:
+ ozone_net:
+ ipam:
+ driver: default
+ config:
+ - subnet: "172.25.0.0/24"
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-config
new file mode 100644
index 0000000..e245b7e
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-config
@@ -0,0 +1,125 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CORE-SITE.XML_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
+CORE-SITE.XML_fs.defaultFS=o3fs://bucket.volume.id1
+OZONE-SITE.XML_ozone.om.service.ids=id1
+OZONE-SITE.XML_ozone.om.internal.service.id=id1
+OZONE-SITE.XML_ozone.om.nodes.id1=om1,om2,om3
+OZONE-SITE.XML_ozone.om.address.id1.om1=om1
+OZONE-SITE.XML_ozone.om.address.id1.om2=om2
+OZONE-SITE.XML_ozone.om.address.id1.om3=om3
+OZONE-SITE.XML_ozone.om.http-address.id1.om1=om1
+OZONE-SITE.XML_ozone.om.http-address.id1.om2=om2
+OZONE-SITE.XML_ozone.om.http-address.id1.om3=om3
+OZONE-SITE.XML_ozone.om.ratis.enable=true
+
+OZONE-SITE.XML_ozone.om.volume.listall.allowed=false
+
+OZONE-SITE.XML_ozone.scm.container.size=1GB
+OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.handler.type=distributed
+OZONE-SITE.XML_ozone.scm.client.address=scm
+OZONE-SITE.XML_hdds.block.token.enabled=true
+OZONE-SITE.XML_ozone.replication=3
+
+OZONE-SITE.XML_recon.om.snapshot.task.interval.delay=1m
+OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
+OZONE-SITE.XML_recon.om.snapshot.task.initial.delay=20s
+OZONE-SITE.XML_ozone.recon.address=recon:9891
+
+OZONE-SITE.XML_ozone.security.enabled=true
+OZONE-SITE.XML_ozone.acl.enabled=true
+OZONE-SITE.XML_ozone.acl.authorizer.class=org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer
+OZONE-SITE.XML_ozone.administrators="testuser/scm@EXAMPLE.COM,testuser/s3g@EXAMPLE.COM"
+
+OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
+HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
+HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
+CORE-SITE.XML_dfs.data.transfer.protection=authentication
+CORE-SITE.XML_hadoop.security.authentication=kerberos
+CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1@$0](.*)s/.*/root/
+CORE-SITE.XML_hadoop.security.key.provider.path=kms://http@kms:9600/kms
+
+
+OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
+OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
+OZONE-SITE.XML_ozone.om.kerberos.principal=om/_HOST@EXAMPLE.COM
+OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
+OZONE-SITE.XML_ozone.recon.kerberos.keytab.file=/etc/security/keytabs/recon.keytab
+OZONE-SITE.XML_ozone.recon.kerberos.principal=recon/recon@EXAMPLE.COM
+
+HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/_HOST@EXAMPLE.COM
+HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
+HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
+HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+
+
+OZONE-SITE.XML_ozone.security.http.kerberos.enabled=true
+OZONE-SITE.XML_ozone.http.filter.initializers=org.apache.hadoop.security.AuthenticationFilterInitializer
+
+OZONE-SITE.XML_ozone.om.http.auth.type=kerberos
+OZONE-SITE.XML_hdds.scm.http.auth.type=kerberos
+OZONE-SITE.XML_hdds.datanode.http.auth.type=kerberos
+OZONE-SITE.XML_ozone.s3g.http.auth.type=kerberos
+OZONE-SITE.XML_ozone.recon.http.auth.type=kerberos
+
+OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal=HTTP/scm@EXAMPLE.COM
+OZONE-SITE.XML_hdds.scm.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.om.http.auth.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
+OZONE-SITE.XML_ozone.om.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
+OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.principal=HTTP/s3g@EXAMPLE.COM
+OZONE-SITE.XML_ozone.recon.http.auth.kerberos.principal=HTTP/recon@EXAMPLE.COM
+OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+
+CORE-SITE.XML_hadoop.http.authentication.simple.anonymous.allowed=false
+CORE-SITE.XML_hadoop.http.authentication.signature.secret.file=/etc/security/http_secret
+CORE-SITE.XML_hadoop.http.authentication.type=kerberos
+CORE-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
+CORE-SITE.XML_hadoop.http.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+
+
+CORE-SITE.XML_hadoop.security.authorization=true
+HADOOP-POLICY.XML_ozone.om.security.client.protocol.acl=*
+HADOOP-POLICY.XML_hdds.security.client.datanode.container.protocol.acl=*
+HADOOP-POLICY.XML_hdds.security.client.scm.container.protocol.acl=*
+HADOOP-POLICY.XML_hdds.security.client.scm.block.protocol.acl=*
+HADOOP-POLICY.XML_hdds.security.client.scm.certificate.protocol.acl=*
+
+HDFS-SITE.XML_rpc.metrics.quantile.enable=true
+HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+
+#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
+#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
+
+OZONE_DATANODE_SECURE_USER=root
+SECURITY_ENABLED=true
+KEYTAB_DIR=/etc/security/keytabs
+KERBEROS_KEYSTORES=hadoop
+KERBEROS_SERVER=kdc
+JAVA_HOME=/usr/lib/jvm/jre
+JSVC_HOME=/usr/bin
+SLEEP_SECONDS=5
+KERBEROS_ENABLED=true
+
+no_proxy=om,scm,recon,s3g,kdc,localhost,127.0.0.1
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/test.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/test.sh
old mode 100644
new mode 100755
similarity index 73%
copy from hadoop-ozone/dist/src/main/compose/ozone-om-ha/test.sh
copy to hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/test.sh
index 3146ae7..8893ef6
--- a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/test.sh
@@ -15,25 +15,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-startOMs(){
- docker-compose exec -T om1 /opt/startOM.sh
- docker-compose exec -T om2 /opt/startOM.sh
- docker-compose exec -T om3 /opt/startOM.sh
-}
-
COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
export COMPOSE_DIR
+export SECURITY_ENABLED=true
+export OM_SERVICE_ID="id1"
+
# shellcheck source=/dev/null
source "$COMPOSE_DIR/../testlib.sh"
start_docker_env
-# Start OMs separately. In this test, the OMs will be stopped and restarted multiple times.
-# So we do not want the container to be tied to the OM process.
-startOMs
+execute_robot_test scm kinit.robot
-execute_robot_test scm omha/testOMHA.robot
+execute_robot_test scm freon
stop_docker_env
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
index 0962675..18d5848 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
@@ -50,6 +50,7 @@
- ../..:/opt/hadoop
ports:
- 9874:9874
+ - 9862:9862
environment:
ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
KERBEROS_KEYTABS: om HTTP
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
index df34dd7..11879f6 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
@@ -15,6 +15,7 @@
# limitations under the License.
CORE-SITE.XML_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
+OZONE-SITE.XML_ozone.om.volume.listall.allowed=false
OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
@@ -28,16 +29,7 @@
OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_hdds.block.token.enabled=true
OZONE-SITE.XML_ozone.replication=3
-OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
-OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
-OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
-OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
-OZONE-SITE.XML_ozone.s3g.keytab.file=/etc/security/keytabs/HTTP.keytab
-OZONE-SITE.XML_ozone.s3g.authentication.kerberos.principal=HTTP/s3g@EXAMPLE.COM
-OZONE-SITE.XML_ozone.recon.authentication.kerberos.principal=HTTP/recon@EXAMPLE.COM
-OZONE-SITE.XML_ozone.recon.http.kerberos.keytab.file=/etc/security/keytabs/HTTP.keytab
-OZONE-SITE.XML_ozone.recon.kerberos.keytab.file=/etc/security/keytabs/recon.keytab
-OZONE-SITE.XML_ozone.recon.kerberos.principal=recon/recon@EXAMPLE.COM
+
OZONE-SITE.XML_recon.om.snapshot.task.interval.delay=1m
OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
OZONE-SITE.XML_recon.om.snapshot.task.initial.delay=20s
@@ -46,29 +38,56 @@
OZONE-SITE.XML_ozone.security.enabled=true
OZONE-SITE.XML_ozone.acl.enabled=true
OZONE-SITE.XML_ozone.acl.authorizer.class=org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer
-OZONE-SITE.XML_ozone.administrators="testuser/scm@EXAMPLE.COM"
-OZONE-SITE.XML_hdds.scm.http.kerberos.principal=HTTP/scm@EXAMPLE.COM
-OZONE-SITE.XML_hdds.scm.http.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-OZONE-SITE.XML_ozone.om.http.kerberos.principal=HTTP/om@EXAMPLE.COM
-OZONE-SITE.XML_ozone.om.http.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/_HOST@EXAMPLE.COM
-HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
-HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
-HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.administrators="testuser/scm@EXAMPLE.COM,testuser/s3g@EXAMPLE.COM"
+
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
CORE-SITE.XML_dfs.data.transfer.protection=authentication
CORE-SITE.XML_hadoop.security.authentication=kerberos
-CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1@$0](.*)s/.*/root/
+CORE-SITE.XML_hadoop.security.auth_to_local="RULE:[2:$1](testuser2.*) RULE:[2:$1@$0](.*)s/.*/root/"
CORE-SITE.XML_hadoop.security.key.provider.path=kms://http@kms:9600/kms
+
+OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
+OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
+OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
+OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
+OZONE-SITE.XML_ozone.recon.kerberos.keytab.file=/etc/security/keytabs/recon.keytab
+OZONE-SITE.XML_ozone.recon.kerberos.principal=recon/recon@EXAMPLE.COM
+
+HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/_HOST@EXAMPLE.COM
+HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
+HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
+HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+
+
+OZONE-SITE.XML_ozone.security.http.kerberos.enabled=true
+OZONE-SITE.XML_ozone.http.filter.initializers=org.apache.hadoop.security.AuthenticationFilterInitializer
+
+OZONE-SITE.XML_ozone.om.http.auth.type=kerberos
+OZONE-SITE.XML_hdds.scm.http.auth.type=kerberos
+OZONE-SITE.XML_hdds.datanode.http.auth.type=kerberos
+OZONE-SITE.XML_ozone.s3g.http.auth.type=kerberos
+OZONE-SITE.XML_ozone.recon.http.auth.type=kerberos
+
+OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal=HTTP/scm@EXAMPLE.COM
+OZONE-SITE.XML_hdds.scm.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.om.http.auth.kerberos.principal=HTTP/om@EXAMPLE.COM
+OZONE-SITE.XML_ozone.om.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
+OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.principal=HTTP/s3g@EXAMPLE.COM
+OZONE-SITE.XML_ozone.recon.http.auth.kerberos.principal=HTTP/recon@EXAMPLE.COM
+OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+
CORE-SITE.XML_hadoop.http.authentication.simple.anonymous.allowed=false
CORE-SITE.XML_hadoop.http.authentication.signature.secret.file=/etc/security/http_secret
CORE-SITE.XML_hadoop.http.authentication.type=kerberos
CORE-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
CORE-SITE.XML_hadoop.http.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-CORE-SITE.XML_hadoop.http.filter.initializers=org.apache.hadoop.security.AuthenticationFilterInitializer
+
CORE-SITE.XML_hadoop.security.authorization=true
HADOOP-POLICY.XML_ozone.om.security.client.protocol.acl=*
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh
index 8b5441a..d1bdd0d 100755
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh
@@ -39,6 +39,8 @@
execute_robot_test scm recon
+execute_robot_test scm spnego
+
stop_docker_env
generate_report
diff --git a/hadoop-ozone/dist/src/main/compose/test-all.sh b/hadoop-ozone/dist/src/main/compose/test-all.sh
index afa5d56..809123b 100755
--- a/hadoop-ozone/dist/src/main/compose/test-all.sh
+++ b/hadoop-ozone/dist/src/main/compose/test-all.sh
@@ -41,7 +41,7 @@
echo "ERROR: Test execution of $(dirname "$test") is FAILED!!!!"
fi
RESULT_DIR="$(dirname "$test")/result"
- cp "$RESULT_DIR"/robot-*.xml "$RESULT_DIR"/docker-*.log "$ALL_RESULT_DIR"/
+ cp "$RESULT_DIR"/robot-*.xml "$RESULT_DIR"/docker-*.log "$RESULT_DIR"/*.out* "$ALL_RESULT_DIR"/
done
rebot -N "smoketests" -d "$SCRIPT_DIR/result" "$SCRIPT_DIR/result/robot-*.xml"
diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh
index e6a97ff..064b19d 100755
--- a/hadoop-ozone/dist/src/main/compose/testlib.sh
+++ b/hadoop-ozone/dist/src/main/compose/testlib.sh
@@ -22,6 +22,11 @@
RESULT_DIR_INSIDE="/tmp/smoketest/$(basename "$COMPOSE_ENV_NAME")/result"
SMOKETEST_DIR_INSIDE="${OZONE_DIR:-/opt/hadoop}/smoketest"
+OM_HA_PARAM=""
+if [[ -n "${OM_SERVICE_ID}" ]]; then
+ OM_HA_PARAM="--om-service-id=${OM_SERVICE_ID}"
+fi
+
## @description create results directory, purging any prior data
create_results_dir() {
#delete previous results
@@ -99,11 +104,14 @@
OUTPUT_PATH="$RESULT_DIR_INSIDE/robot-$OUTPUT_NAME.xml"
# shellcheck disable=SC2068
docker-compose -f "$COMPOSE_FILE" exec -T "$CONTAINER" mkdir -p "$RESULT_DIR_INSIDE" \
- && docker-compose -f "$COMPOSE_FILE" exec -T -e SECURITY_ENABLED="${SECURITY_ENABLED}" -e OM_HA_PARAM="${OM_HA_PARAM}" "$CONTAINER" python -m robot ${ARGUMENTS[@]} --log NONE -N "$TEST_NAME" --report NONE "${OZONE_ROBOT_OPTS[@]}" --output "$OUTPUT_PATH" "$SMOKETEST_DIR_INSIDE/$TEST"
+ && docker-compose -f "$COMPOSE_FILE" exec -T -e SECURITY_ENABLED="${SECURITY_ENABLED}" -e OM_HA_PARAM="${OM_HA_PARAM}" -e OM_SERVICE_ID="${OM_SERVICE_ID}" "$CONTAINER" robot ${ARGUMENTS[@]} --log NONE -N "$TEST_NAME" --report NONE "${OZONE_ROBOT_OPTS[@]}" --output "$OUTPUT_PATH" "$SMOKETEST_DIR_INSIDE/$TEST"
local -i rc=$?
FULL_CONTAINER_NAME=$(docker-compose -f "$COMPOSE_FILE" ps | grep "_${CONTAINER}_" | head -n 1 | awk '{print $1}')
docker cp "$FULL_CONTAINER_NAME:$OUTPUT_PATH" "$RESULT_DIR/"
+
+ copy_daemon_logs
+
set -e
if [[ ${rc} -gt 0 ]]; then
@@ -113,6 +121,16 @@
return ${rc}
}
+## @description Copy any 'out' files for daemon processes to the result dir
+copy_daemon_logs() {
+ local c f
+ for c in $(docker-compose -f "$COMPOSE_FILE" ps | grep "^${COMPOSE_ENV_NAME}_" | awk '{print $1}'); do
+ for f in $(docker exec "${c}" ls -1 /var/log/hadoop | grep -F '.out'); do
+ docker cp "${c}:/var/log/hadoop/${f}" "$RESULT_DIR/"
+ done
+ done
+}
+
## @description Execute specific command in docker container
## @param container name
@@ -179,6 +197,13 @@
fi
}
+## @description Removes the given docker images if configured not to keep them (via KEEP_IMAGE=false)
+cleanup_docker_images() {
+ if [[ "${KEEP_IMAGE:-true}" == false ]]; then
+ docker image rm "$@"
+ fi
+}
+
## @description Generate robot framework reports based on the saved results.
generate_report(){
diff --git a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt
index 6e661af..037df79 100644
--- a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt
+++ b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt
@@ -225,14 +225,6 @@
org.codehaus.woodstox:stax2-api
-
-GPL with classpath exception
-=====================
-
- org.openjdk.jmh:jmh-core
- org.openjdk.jmh:jmh-generator-annprocess
-
-
CDDL
=====================
diff --git a/hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-org.openjdk.jmh-jmh-core.txt b/hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-org.openjdk.jmh-jmh-core.txt
deleted file mode 100644
index b40a0f4..0000000
--- a/hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-org.openjdk.jmh-jmh-core.txt
+++ /dev/null
@@ -1,347 +0,0 @@
-The GNU General Public License (GPL)
-
-Version 2, June 1991
-
-Copyright (C) 1989, 1991 Free Software Foundation, Inc.
-59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-
-Everyone is permitted to copy and distribute verbatim copies of this license
-document, but changing it is not allowed.
-
-Preamble
-
-The licenses for most software are designed to take away your freedom to share
-and change it. By contrast, the GNU General Public License is intended to
-guarantee your freedom to share and change free software--to make sure the
-software is free for all its users. This General Public License applies to
-most of the Free Software Foundation's software and to any other program whose
-authors commit to using it. (Some other Free Software Foundation software is
-covered by the GNU Library General Public License instead.) You can apply it to
-your programs, too.
-
-When we speak of free software, we are referring to freedom, not price. Our
-General Public Licenses are designed to make sure that you have the freedom to
-distribute copies of free software (and charge for this service if you wish),
-that you receive source code or can get it if you want it, that you can change
-the software or use pieces of it in new free programs; and that you know you
-can do these things.
-
-To protect your rights, we need to make restrictions that forbid anyone to deny
-you these rights or to ask you to surrender the rights. These restrictions
-translate to certain responsibilities for you if you distribute copies of the
-software, or if you modify it.
-
-For example, if you distribute copies of such a program, whether gratis or for
-a fee, you must give the recipients all the rights that you have. You must
-make sure that they, too, receive or can get the source code. And you must
-show them these terms so they know their rights.
-
-We protect your rights with two steps: (1) copyright the software, and (2)
-offer you this license which gives you legal permission to copy, distribute
-and/or modify the software.
-
-Also, for each author's protection and ours, we want to make certain that
-everyone understands that there is no warranty for this free software. If the
-software is modified by someone else and passed on, we want its recipients to
-know that what they have is not the original, so that any problems introduced
-by others will not reflect on the original authors' reputations.
-
-Finally, any free program is threatened constantly by software patents. We
-wish to avoid the danger that redistributors of a free program will
-individually obtain patent licenses, in effect making the program proprietary.
-To prevent this, we have made it clear that any patent must be licensed for
-everyone's free use or not licensed at all.
-
-The precise terms and conditions for copying, distribution and modification
-follow.
-
-TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
-
-0. This License applies to any program or other work which contains a notice
-placed by the copyright holder saying it may be distributed under the terms of
-this General Public License. The "Program", below, refers to any such program
-or work, and a "work based on the Program" means either the Program or any
-derivative work under copyright law: that is to say, a work containing the
-Program or a portion of it, either verbatim or with modifications and/or
-translated into another language. (Hereinafter, translation is included
-without limitation in the term "modification".) Each licensee is addressed as
-"you".
-
-Activities other than copying, distribution and modification are not covered by
-this License; they are outside its scope. The act of running the Program is
-not restricted, and the output from the Program is covered only if its contents
-constitute a work based on the Program (independent of having been made by
-running the Program). Whether that is true depends on what the Program does.
-
-1. You may copy and distribute verbatim copies of the Program's source code as
-you receive it, in any medium, provided that you conspicuously and
-appropriately publish on each copy an appropriate copyright notice and
-disclaimer of warranty; keep intact all the notices that refer to this License
-and to the absence of any warranty; and give any other recipients of the
-Program a copy of this License along with the Program.
-
-You may charge a fee for the physical act of transferring a copy, and you may
-at your option offer warranty protection in exchange for a fee.
-
-2. You may modify your copy or copies of the Program or any portion of it, thus
-forming a work based on the Program, and copy and distribute such modifications
-or work under the terms of Section 1 above, provided that you also meet all of
-these conditions:
-
- a) You must cause the modified files to carry prominent notices stating
- that you changed the files and the date of any change.
-
- b) You must cause any work that you distribute or publish, that in whole or
- in part contains or is derived from the Program or any part thereof, to be
- licensed as a whole at no charge to all third parties under the terms of
- this License.
-
- c) If the modified program normally reads commands interactively when run,
- you must cause it, when started running for such interactive use in the
- most ordinary way, to print or display an announcement including an
- appropriate copyright notice and a notice that there is no warranty (or
- else, saying that you provide a warranty) and that users may redistribute
- the program under these conditions, and telling the user how to view a copy
- of this License. (Exception: if the Program itself is interactive but does
- not normally print such an announcement, your work based on the Program is
- not required to print an announcement.)
-
-These requirements apply to the modified work as a whole. If identifiable
-sections of that work are not derived from the Program, and can be reasonably
-considered independent and separate works in themselves, then this License, and
-its terms, do not apply to those sections when you distribute them as separate
-works. But when you distribute the same sections as part of a whole which is a
-work based on the Program, the distribution of the whole must be on the terms
-of this License, whose permissions for other licensees extend to the entire
-whole, and thus to each and every part regardless of who wrote it.
-
-Thus, it is not the intent of this section to claim rights or contest your
-rights to work written entirely by you; rather, the intent is to exercise the
-right to control the distribution of derivative or collective works based on
-the Program.
-
-In addition, mere aggregation of another work not based on the Program with the
-Program (or with a work based on the Program) on a volume of a storage or
-distribution medium does not bring the other work under the scope of this
-License.
-
-3. You may copy and distribute the Program (or a work based on it, under
-Section 2) in object code or executable form under the terms of Sections 1 and
-2 above provided that you also do one of the following:
-
- a) Accompany it with the complete corresponding machine-readable source
- code, which must be distributed under the terms of Sections 1 and 2 above
- on a medium customarily used for software interchange; or,
-
- b) Accompany it with a written offer, valid for at least three years, to
- give any third party, for a charge no more than your cost of physically
- performing source distribution, a complete machine-readable copy of the
- corresponding source code, to be distributed under the terms of Sections 1
- and 2 above on a medium customarily used for software interchange; or,
-
- c) Accompany it with the information you received as to the offer to
- distribute corresponding source code. (This alternative is allowed only
- for noncommercial distribution and only if you received the program in
- object code or executable form with such an offer, in accord with
- Subsection b above.)
-
-The source code for a work means the preferred form of the work for making
-modifications to it. For an executable work, complete source code means all
-the source code for all modules it contains, plus any associated interface
-definition files, plus the scripts used to control compilation and installation
-of the executable. However, as a special exception, the source code
-distributed need not include anything that is normally distributed (in either
-source or binary form) with the major components (compiler, kernel, and so on)
-of the operating system on which the executable runs, unless that component
-itself accompanies the executable.
-
-If distribution of executable or object code is made by offering access to copy
-from a designated place, then offering equivalent access to copy the source
-code from the same place counts as distribution of the source code, even though
-third parties are not compelled to copy the source along with the object code.
-
-4. You may not copy, modify, sublicense, or distribute the Program except as
-expressly provided under this License. Any attempt otherwise to copy, modify,
-sublicense or distribute the Program is void, and will automatically terminate
-your rights under this License. However, parties who have received copies, or
-rights, from you under this License will not have their licenses terminated so
-long as such parties remain in full compliance.
-
-5. You are not required to accept this License, since you have not signed it.
-However, nothing else grants you permission to modify or distribute the Program
-or its derivative works. These actions are prohibited by law if you do not
-accept this License. Therefore, by modifying or distributing the Program (or
-any work based on the Program), you indicate your acceptance of this License to
-do so, and all its terms and conditions for copying, distributing or modifying
-the Program or works based on it.
-
-6. Each time you redistribute the Program (or any work based on the Program),
-the recipient automatically receives a license from the original licensor to
-copy, distribute or modify the Program subject to these terms and conditions.
-You may not impose any further restrictions on the recipients' exercise of the
-rights granted herein. You are not responsible for enforcing compliance by
-third parties to this License.
-
-7. If, as a consequence of a court judgment or allegation of patent
-infringement or for any other reason (not limited to patent issues), conditions
-are imposed on you (whether by court order, agreement or otherwise) that
-contradict the conditions of this License, they do not excuse you from the
-conditions of this License. If you cannot distribute so as to satisfy
-simultaneously your obligations under this License and any other pertinent
-obligations, then as a consequence you may not distribute the Program at all.
-For example, if a patent license would not permit royalty-free redistribution
-of the Program by all those who receive copies directly or indirectly through
-you, then the only way you could satisfy both it and this License would be to
-refrain entirely from distribution of the Program.
-
-If any portion of this section is held invalid or unenforceable under any
-particular circumstance, the balance of the section is intended to apply and
-the section as a whole is intended to apply in other circumstances.
-
-It is not the purpose of this section to induce you to infringe any patents or
-other property right claims or to contest validity of any such claims; this
-section has the sole purpose of protecting the integrity of the free software
-distribution system, which is implemented by public license practices. Many
-people have made generous contributions to the wide range of software
-distributed through that system in reliance on consistent application of that
-system; it is up to the author/donor to decide if he or she is willing to
-distribute software through any other system and a licensee cannot impose that
-choice.
-
-This section is intended to make thoroughly clear what is believed to be a
-consequence of the rest of this License.
-
-8. If the distribution and/or use of the Program is restricted in certain
-countries either by patents or by copyrighted interfaces, the original
-copyright holder who places the Program under this License may add an explicit
-geographical distribution limitation excluding those countries, so that
-distribution is permitted only in or among countries not thus excluded. In
-such case, this License incorporates the limitation as if written in the body
-of this License.
-
-9. The Free Software Foundation may publish revised and/or new versions of the
-General Public License from time to time. Such new versions will be similar in
-spirit to the present version, but may differ in detail to address new problems
-or concerns.
-
-Each version is given a distinguishing version number. If the Program
-specifies a version number of this License which applies to it and "any later
-version", you have the option of following the terms and conditions either of
-that version or of any later version published by the Free Software Foundation.
-If the Program does not specify a version number of this License, you may
-choose any version ever published by the Free Software Foundation.
-
-10. If you wish to incorporate parts of the Program into other free programs
-whose distribution conditions are different, write to the author to ask for
-permission. For software which is copyrighted by the Free Software Foundation,
-write to the Free Software Foundation; we sometimes make exceptions for this.
-Our decision will be guided by the two goals of preserving the free status of
-all derivatives of our free software and of promoting the sharing and reuse of
-software generally.
-
-NO WARRANTY
-
-11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR
-THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE
-STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE
-PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED,
-INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
-FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND
-PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE,
-YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
-12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL
-ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE
-PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
-GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR
-INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA
-BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
-FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER
-OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
-
-END OF TERMS AND CONDITIONS
-
-How to Apply These Terms to Your New Programs
-
-If you develop a new program, and you want it to be of the greatest possible
-use to the public, the best way to achieve this is to make it free software
-which everyone can redistribute and change under these terms.
-
-To do so, attach the following notices to the program. It is safest to attach
-them to the start of each source file to most effectively convey the exclusion
-of warranty; and each file should have at least the "copyright" line and a
-pointer to where the full notice is found.
-
- One line to give the program's name and a brief idea of what it does.
-
- Copyright (C) <year> <name of author>
-
- This program is free software; you can redistribute it and/or modify it
- under the terms of the GNU General Public License as published by the Free
- Software Foundation; either version 2 of the License, or (at your option)
- any later version.
-
- This program is distributed in the hope that it will be useful, but WITHOUT
- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- more details.
-
- You should have received a copy of the GNU General Public License along
- with this program; if not, write to the Free Software Foundation, Inc., 59
- Temple Place, Suite 330, Boston, MA 02111-1307 USA
-
-Also add information on how to contact you by electronic and paper mail.
-
-If the program is interactive, make it output a short notice like this when it
-starts in an interactive mode:
-
- Gnomovision version 69, Copyright (C) year name of author Gnomovision comes
- with ABSOLUTELY NO WARRANTY; for details type 'show w'. This is free
- software, and you are welcome to redistribute it under certain conditions;
- type 'show c' for details.
-
-The hypothetical commands 'show w' and 'show c' should show the appropriate
-parts of the General Public License. Of course, the commands you use may be
-called something other than 'show w' and 'show c'; they could even be
-mouse-clicks or menu items--whatever suits your program.
-
-You should also get your employer (if you work as a programmer) or your school,
-if any, to sign a "copyright disclaimer" for the program, if necessary. Here
-is a sample; alter the names:
-
- Yoyodyne, Inc., hereby disclaims all copyright interest in the program
- 'Gnomovision' (which makes passes at compilers) written by James Hacker.
-
- signature of Ty Coon, 1 April 1989
-
- Ty Coon, President of Vice
-
-This General Public License does not permit incorporating your program into
-proprietary programs. If your program is a subroutine library, you may
-consider it more useful to permit linking proprietary applications with the
-library. If this is what you want to do, use the GNU Library General Public
-License instead of this License.
-
-
-"CLASSPATH" EXCEPTION TO THE GPL
-
-Certain source files distributed by Oracle America and/or its affiliates are
-subject to the following clarification and special exception to the GPL, but
-only where Oracle has expressly included in the particular source file's header
-the words "Oracle designates this particular file as subject to the "Classpath"
-exception as provided by Oracle in the LICENSE file that accompanied this code."
-
- Linking this library statically or dynamically with other modules is making
- a combined work based on this library. Thus, the terms and conditions of
- the GNU General Public License cover the whole combination.
-
- As a special exception, the copyright holders of this library give you
- permission to link this library with independent modules to produce an
- executable, regardless of the license terms of these independent modules,
- and to copy and distribute the resulting executable under terms of your
- choice, provided that you also meet, for each linked independent module,
- the terms and conditions of the license of that module. An independent
- module is a module which is not derived from or based on this library. If
- you modify this library, you may extend this exception to your version of
- the library, but you are not obligated to do so. If you do not wish to do
- so, delete this exception statement from your version.
diff --git a/hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-org.openjdk.jmh-jmh-generator-annprocess.txt b/hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-org.openjdk.jmh-jmh-generator-annprocess.txt
deleted file mode 100644
index b40a0f4..0000000
--- a/hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-org.openjdk.jmh-jmh-generator-annprocess.txt
+++ /dev/null
@@ -1,347 +0,0 @@
-The GNU General Public License (GPL)
-
-Version 2, June 1991
-
-Copyright (C) 1989, 1991 Free Software Foundation, Inc.
-59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-
-Everyone is permitted to copy and distribute verbatim copies of this license
-document, but changing it is not allowed.
-
-Preamble
-
-The licenses for most software are designed to take away your freedom to share
-and change it. By contrast, the GNU General Public License is intended to
-guarantee your freedom to share and change free software--to make sure the
-software is free for all its users. This General Public License applies to
-most of the Free Software Foundation's software and to any other program whose
-authors commit to using it. (Some other Free Software Foundation software is
-covered by the GNU Library General Public License instead.) You can apply it to
-your programs, too.
-
-When we speak of free software, we are referring to freedom, not price. Our
-General Public Licenses are designed to make sure that you have the freedom to
-distribute copies of free software (and charge for this service if you wish),
-that you receive source code or can get it if you want it, that you can change
-the software or use pieces of it in new free programs; and that you know you
-can do these things.
-
-To protect your rights, we need to make restrictions that forbid anyone to deny
-you these rights or to ask you to surrender the rights. These restrictions
-translate to certain responsibilities for you if you distribute copies of the
-software, or if you modify it.
-
-For example, if you distribute copies of such a program, whether gratis or for
-a fee, you must give the recipients all the rights that you have. You must
-make sure that they, too, receive or can get the source code. And you must
-show them these terms so they know their rights.
-
-We protect your rights with two steps: (1) copyright the software, and (2)
-offer you this license which gives you legal permission to copy, distribute
-and/or modify the software.
-
-Also, for each author's protection and ours, we want to make certain that
-everyone understands that there is no warranty for this free software. If the
-software is modified by someone else and passed on, we want its recipients to
-know that what they have is not the original, so that any problems introduced
-by others will not reflect on the original authors' reputations.
-
-Finally, any free program is threatened constantly by software patents. We
-wish to avoid the danger that redistributors of a free program will
-individually obtain patent licenses, in effect making the program proprietary.
-To prevent this, we have made it clear that any patent must be licensed for
-everyone's free use or not licensed at all.
-
-The precise terms and conditions for copying, distribution and modification
-follow.
-
-TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
-
-0. This License applies to any program or other work which contains a notice
-placed by the copyright holder saying it may be distributed under the terms of
-this General Public License. The "Program", below, refers to any such program
-or work, and a "work based on the Program" means either the Program or any
-derivative work under copyright law: that is to say, a work containing the
-Program or a portion of it, either verbatim or with modifications and/or
-translated into another language. (Hereinafter, translation is included
-without limitation in the term "modification".) Each licensee is addressed as
-"you".
-
-Activities other than copying, distribution and modification are not covered by
-this License; they are outside its scope. The act of running the Program is
-not restricted, and the output from the Program is covered only if its contents
-constitute a work based on the Program (independent of having been made by
-running the Program). Whether that is true depends on what the Program does.
-
-1. You may copy and distribute verbatim copies of the Program's source code as
-you receive it, in any medium, provided that you conspicuously and
-appropriately publish on each copy an appropriate copyright notice and
-disclaimer of warranty; keep intact all the notices that refer to this License
-and to the absence of any warranty; and give any other recipients of the
-Program a copy of this License along with the Program.
-
-You may charge a fee for the physical act of transferring a copy, and you may
-at your option offer warranty protection in exchange for a fee.
-
-2. You may modify your copy or copies of the Program or any portion of it, thus
-forming a work based on the Program, and copy and distribute such modifications
-or work under the terms of Section 1 above, provided that you also meet all of
-these conditions:
-
- a) You must cause the modified files to carry prominent notices stating
- that you changed the files and the date of any change.
-
- b) You must cause any work that you distribute or publish, that in whole or
- in part contains or is derived from the Program or any part thereof, to be
- licensed as a whole at no charge to all third parties under the terms of
- this License.
-
- c) If the modified program normally reads commands interactively when run,
- you must cause it, when started running for such interactive use in the
- most ordinary way, to print or display an announcement including an
- appropriate copyright notice and a notice that there is no warranty (or
- else, saying that you provide a warranty) and that users may redistribute
- the program under these conditions, and telling the user how to view a copy
- of this License. (Exception: if the Program itself is interactive but does
- not normally print such an announcement, your work based on the Program is
- not required to print an announcement.)
-
-These requirements apply to the modified work as a whole. If identifiable
-sections of that work are not derived from the Program, and can be reasonably
-considered independent and separate works in themselves, then this License, and
-its terms, do not apply to those sections when you distribute them as separate
-works. But when you distribute the same sections as part of a whole which is a
-work based on the Program, the distribution of the whole must be on the terms
-of this License, whose permissions for other licensees extend to the entire
-whole, and thus to each and every part regardless of who wrote it.
-
-Thus, it is not the intent of this section to claim rights or contest your
-rights to work written entirely by you; rather, the intent is to exercise the
-right to control the distribution of derivative or collective works based on
-the Program.
-
-In addition, mere aggregation of another work not based on the Program with the
-Program (or with a work based on the Program) on a volume of a storage or
-distribution medium does not bring the other work under the scope of this
-License.
-
-3. You may copy and distribute the Program (or a work based on it, under
-Section 2) in object code or executable form under the terms of Sections 1 and
-2 above provided that you also do one of the following:
-
- a) Accompany it with the complete corresponding machine-readable source
- code, which must be distributed under the terms of Sections 1 and 2 above
- on a medium customarily used for software interchange; or,
-
- b) Accompany it with a written offer, valid for at least three years, to
- give any third party, for a charge no more than your cost of physically
- performing source distribution, a complete machine-readable copy of the
- corresponding source code, to be distributed under the terms of Sections 1
- and 2 above on a medium customarily used for software interchange; or,
-
- c) Accompany it with the information you received as to the offer to
- distribute corresponding source code. (This alternative is allowed only
- for noncommercial distribution and only if you received the program in
- object code or executable form with such an offer, in accord with
- Subsection b above.)
-
-The source code for a work means the preferred form of the work for making
-modifications to it. For an executable work, complete source code means all
-the source code for all modules it contains, plus any associated interface
-definition files, plus the scripts used to control compilation and installation
-of the executable. However, as a special exception, the source code
-distributed need not include anything that is normally distributed (in either
-source or binary form) with the major components (compiler, kernel, and so on)
-of the operating system on which the executable runs, unless that component
-itself accompanies the executable.
-
-If distribution of executable or object code is made by offering access to copy
-from a designated place, then offering equivalent access to copy the source
-code from the same place counts as distribution of the source code, even though
-third parties are not compelled to copy the source along with the object code.
-
-4. You may not copy, modify, sublicense, or distribute the Program except as
-expressly provided under this License. Any attempt otherwise to copy, modify,
-sublicense or distribute the Program is void, and will automatically terminate
-your rights under this License. However, parties who have received copies, or
-rights, from you under this License will not have their licenses terminated so
-long as such parties remain in full compliance.
-
-5. You are not required to accept this License, since you have not signed it.
-However, nothing else grants you permission to modify or distribute the Program
-or its derivative works. These actions are prohibited by law if you do not
-accept this License. Therefore, by modifying or distributing the Program (or
-any work based on the Program), you indicate your acceptance of this License to
-do so, and all its terms and conditions for copying, distributing or modifying
-the Program or works based on it.
-
-6. Each time you redistribute the Program (or any work based on the Program),
-the recipient automatically receives a license from the original licensor to
-copy, distribute or modify the Program subject to these terms and conditions.
-You may not impose any further restrictions on the recipients' exercise of the
-rights granted herein. You are not responsible for enforcing compliance by
-third parties to this License.
-
-7. If, as a consequence of a court judgment or allegation of patent
-infringement or for any other reason (not limited to patent issues), conditions
-are imposed on you (whether by court order, agreement or otherwise) that
-contradict the conditions of this License, they do not excuse you from the
-conditions of this License. If you cannot distribute so as to satisfy
-simultaneously your obligations under this License and any other pertinent
-obligations, then as a consequence you may not distribute the Program at all.
-For example, if a patent license would not permit royalty-free redistribution
-of the Program by all those who receive copies directly or indirectly through
-you, then the only way you could satisfy both it and this License would be to
-refrain entirely from distribution of the Program.
-
-If any portion of this section is held invalid or unenforceable under any
-particular circumstance, the balance of the section is intended to apply and
-the section as a whole is intended to apply in other circumstances.
-
-It is not the purpose of this section to induce you to infringe any patents or
-other property right claims or to contest validity of any such claims; this
-section has the sole purpose of protecting the integrity of the free software
-distribution system, which is implemented by public license practices. Many
-people have made generous contributions to the wide range of software
-distributed through that system in reliance on consistent application of that
-system; it is up to the author/donor to decide if he or she is willing to
-distribute software through any other system and a licensee cannot impose that
-choice.
-
-This section is intended to make thoroughly clear what is believed to be a
-consequence of the rest of this License.
-
-8. If the distribution and/or use of the Program is restricted in certain
-countries either by patents or by copyrighted interfaces, the original
-copyright holder who places the Program under this License may add an explicit
-geographical distribution limitation excluding those countries, so that
-distribution is permitted only in or among countries not thus excluded. In
-such case, this License incorporates the limitation as if written in the body
-of this License.
-
-9. The Free Software Foundation may publish revised and/or new versions of the
-General Public License from time to time. Such new versions will be similar in
-spirit to the present version, but may differ in detail to address new problems
-or concerns.
-
-Each version is given a distinguishing version number. If the Program
-specifies a version number of this License which applies to it and "any later
-version", you have the option of following the terms and conditions either of
-that version or of any later version published by the Free Software Foundation.
-If the Program does not specify a version number of this License, you may
-choose any version ever published by the Free Software Foundation.
-
-10. If you wish to incorporate parts of the Program into other free programs
-whose distribution conditions are different, write to the author to ask for
-permission. For software which is copyrighted by the Free Software Foundation,
-write to the Free Software Foundation; we sometimes make exceptions for this.
-Our decision will be guided by the two goals of preserving the free status of
-all derivatives of our free software and of promoting the sharing and reuse of
-software generally.
-
-NO WARRANTY
-
-11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR
-THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE
-STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE
-PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED,
-INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
-FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND
-PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE,
-YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
-12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL
-ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE
-PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
-GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR
-INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA
-BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
-FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER
-OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
-
-END OF TERMS AND CONDITIONS
-
-How to Apply These Terms to Your New Programs
-
-If you develop a new program, and you want it to be of the greatest possible
-use to the public, the best way to achieve this is to make it free software
-which everyone can redistribute and change under these terms.
-
-To do so, attach the following notices to the program. It is safest to attach
-them to the start of each source file to most effectively convey the exclusion
-of warranty; and each file should have at least the "copyright" line and a
-pointer to where the full notice is found.
-
- One line to give the program's name and a brief idea of what it does.
-
- Copyright (C) <year> <name of author>
-
- This program is free software; you can redistribute it and/or modify it
- under the terms of the GNU General Public License as published by the Free
- Software Foundation; either version 2 of the License, or (at your option)
- any later version.
-
- This program is distributed in the hope that it will be useful, but WITHOUT
- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- more details.
-
- You should have received a copy of the GNU General Public License along
- with this program; if not, write to the Free Software Foundation, Inc., 59
- Temple Place, Suite 330, Boston, MA 02111-1307 USA
-
-Also add information on how to contact you by electronic and paper mail.
-
-If the program is interactive, make it output a short notice like this when it
-starts in an interactive mode:
-
- Gnomovision version 69, Copyright (C) year name of author Gnomovision comes
- with ABSOLUTELY NO WARRANTY; for details type 'show w'. This is free
- software, and you are welcome to redistribute it under certain conditions;
- type 'show c' for details.
-
-The hypothetical commands 'show w' and 'show c' should show the appropriate
-parts of the General Public License. Of course, the commands you use may be
-called something other than 'show w' and 'show c'; they could even be
-mouse-clicks or menu items--whatever suits your program.
-
-You should also get your employer (if you work as a programmer) or your school,
-if any, to sign a "copyright disclaimer" for the program, if necessary. Here
-is a sample; alter the names:
-
- Yoyodyne, Inc., hereby disclaims all copyright interest in the program
- 'Gnomovision' (which makes passes at compilers) written by James Hacker.
-
- signature of Ty Coon, 1 April 1989
-
- Ty Coon, President of Vice
-
-This General Public License does not permit incorporating your program into
-proprietary programs. If your program is a subroutine library, you may
-consider it more useful to permit linking proprietary applications with the
-library. If this is what you want to do, use the GNU Library General Public
-License instead of this License.
-
-
-"CLASSPATH" EXCEPTION TO THE GPL
-
-Certain source files distributed by Oracle America and/or its affiliates are
-subject to the following clarification and special exception to the GPL, but
-only where Oracle has expressly included in the particular source file's header
-the words "Oracle designates this particular file as subject to the "Classpath"
-exception as provided by Oracle in the LICENSE file that accompanied this code."
-
- Linking this library statically or dynamically with other modules is making
- a combined work based on this library. Thus, the terms and conditions of
- the GNU General Public License cover the whole combination.
-
- As a special exception, the copyright holders of this library give you
- permission to link this library with independent modules to produce an
- executable, regardless of the license terms of these independent modules,
- and to copy and distribute the resulting executable under terms of your
- choice, provided that you also meet, for each linked independent module,
- the terms and conditions of the license of that module. An independent
- module is a module which is not derived from or based on this library. If
- you modify this library, you may extend this exception to your version of
- the library, but you are not obligated to do so. If you do not wish to do
- so, delete this exception statement from your version.
diff --git a/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell.robot b/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell.robot
index bf5f3da..e358ea2 100644
--- a/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell.robot
@@ -113,7 +113,11 @@
Execute ozone sh key put ${protocol}${server}/${volume}/bb1/key1 /opt/hadoop/NOTICE.txt
Execute rm -f /tmp/NOTICE.txt.1
Execute ozone sh key get ${protocol}${server}/${volume}/bb1/key1 /tmp/NOTICE.txt.1
- Execute ls -l /tmp/NOTICE.txt.1
+ Execute diff -q /opt/hadoop/NOTICE.txt /tmp/NOTICE.txt.1
+ ${result} = Execute And Ignore Error ozone sh key get ${protocol}${server}/${volume}/bb1/key1 /tmp/NOTICE.txt.1
+ Should Contain ${result} NOTICE.txt.1 exists
+ ${result} = Execute ozone sh key get --force ${protocol}${server}/${volume}/bb1/key1 /tmp/NOTICE.txt.1
+ Should Not Contain ${result} NOTICE.txt.1 exists
${result} = Execute ozone sh key info ${protocol}${server}/${volume}/bb1/key1 | jq -r '. | select(.name=="key1")'
Should contain ${result} creationTime
${result} = Execute ozone sh key list ${protocol}${server}/${volume}/bb1 | jq -r '. | select(.name=="key1") | .name'
diff --git a/hadoop-ozone/dist/src/main/smoketest/commonlib.robot b/hadoop-ozone/dist/src/main/smoketest/commonlib.robot
index 64ac275..133c63e 100644
--- a/hadoop-ozone/dist/src/main/smoketest/commonlib.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/commonlib.robot
@@ -21,6 +21,7 @@
*** Variables ***
${SECURITY_ENABLED} %{SECURITY_ENABLED}
${OM_HA_PARAM} %{OM_HA_PARAM}
+${OM_SERVICE_ID} %{OM_SERVICE_ID}
*** Keywords ***
Execute
diff --git a/hadoop-ozone/dist/src/main/smoketest/createbucketenv.robot b/hadoop-ozone/dist/src/main/smoketest/createbucketenv.robot
index da97001..c89abfc 100644
--- a/hadoop-ozone/dist/src/main/smoketest/createbucketenv.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/createbucketenv.robot
@@ -21,7 +21,7 @@
*** Variables ***
-${volume} vol1
+${volume} volume1
${bucket} bucket1
diff --git a/hadoop-ozone/dist/src/main/smoketest/createmrenv.robot b/hadoop-ozone/dist/src/main/smoketest/createmrenv.robot
index 2f93e6c..579fdde 100644
--- a/hadoop-ozone/dist/src/main/smoketest/createmrenv.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/createmrenv.robot
@@ -21,7 +21,7 @@
*** Variables ***
-${volume} vol1
+${volume} volume1
${bucket} bucket1
@@ -40,7 +40,7 @@
Run Keyword if "BUCKET_NOT_FOUND" in """${result}""" Create bucket
${result} = Execute ozone sh bucket info /${volume}/${bucket}
Should not contain ${result} NOT_FOUND
- Execute ozone sh key put /vol1/bucket1/key1 LICENSE.txt
+ Execute ozone sh key put /volume1/bucket1/key1 LICENSE.txt
Create user dir for hadoop
Execute ozone fs -mkdir /user
diff --git a/hadoop-ozone/dist/src/main/smoketest/csi.robot b/hadoop-ozone/dist/src/main/smoketest/csi.robot
new file mode 100644
index 0000000..e93f256
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/csi.robot
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation Smoketest Ozone CSI service
+Library OperatingSystem
+Library BuiltIn
+Library String
+Resource commonlib.robot
+Test Timeout 1 minutes
+
+*** Keywords ***
+CSI Socket check
+ Execute find -name /tmp/csi.sock
+
+*** Test Cases ***
+Check if CSI server is started
+ Wait Until Keyword Succeeds 3min 1sec CSI Socket check
+
+Test CSI identitiy service
+ ${result} = Execute csc -e unix:///tmp/csi.sock identity plugin-info
+ Should Contain ${result} org.apache.hadoop.ozone
diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot
new file mode 100644
index 0000000..39e561a
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation Test ozone Debug CLI
+Library OperatingSystem
+Resource ../commonlib.robot
+Test Timeout 2 minute
+Suite Setup Write key
+*** Variables ***
+
+*** Keywords ***
+Write key
+ Execute ozone sh volume create o3://om/vol1 --quota 100TB
+ Execute ozone sh bucket create o3://om/vol1/bucket1
+ Execute ozone sh key put o3://om/vol1/bucket1/debugKey /opt/hadoop/NOTICE.txt
+
+*** Test Cases ***
+Test ozone debug
+ ${result} = Execute ozone debug chunkinfo o3://om/vol1/bucket1/debugKey | jq -r '.[]'
+ Should contain ${result} files
+ ${result} = Execute ozone debug chunkinfo o3://om/vol1/bucket1/debugKey | jq -r '.[].files[0]'
+ File Should Exist ${result}
+
diff --git a/hadoop-ozone/dist/src/main/smoketest/mapreduce.robot b/hadoop-ozone/dist/src/main/smoketest/mapreduce.robot
index 789ec4f..c1e3285 100644
--- a/hadoop-ozone/dist/src/main/smoketest/mapreduce.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/mapreduce.robot
@@ -21,7 +21,7 @@
*** Variables ***
-${volume} vol1
+${volume} volume1
${bucket} bucket1
${hadoop.version} 3.2.0
@@ -33,5 +33,5 @@
Execute WordCount
${random} Generate Random String 2 [NUMBERS]
- ${output} = Execute yarn jar ./share/hadoop/mapreduce/hadoop-mapreduce-examples-${hadoop.version}.jar wordcount o3fs://bucket1.vol1/key1 o3fs://bucket1.vol1/key1-${random}.count
+ ${output} = Execute yarn jar ./share/hadoop/mapreduce/hadoop-mapreduce-examples-${hadoop.version}.jar wordcount o3fs://bucket1.volume1/key1 o3fs://bucket1.volume1/key1-${random}.count
Should Contain ${output} completed successfully
diff --git a/hadoop-ozone/dist/src/main/smoketest/omha/testOMHA.robot b/hadoop-ozone/dist/src/main/smoketest/omha/testOMHA.robot
index 2ca618c..9261a4b 100644
--- a/hadoop-ozone/dist/src/main/smoketest/omha/testOMHA.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/omha/testOMHA.robot
@@ -26,7 +26,7 @@
${HOST} om1
${USERNAME} hadoop
${PUBLIC_KEY} /opt/.ssh/id_rsa
-${OM_SERVICE_ID} omservice
+${OM_SERVICE_ID} %{OM_SERVICE_ID}
${OZONE_LOG_DIR} /ozone/logs/
${RATIS_DIR} /data/metadata/ratis
${VOLUME} volume1
diff --git a/hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot b/hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot
index 3336b39..df9f3af 100644
--- a/hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot
@@ -28,6 +28,6 @@
Test hadoop dfs
${random} = Generate Random String 5 [NUMBERS]
- ${result} = Execute hdfs dfs -put /opt/hadoop/NOTICE.txt o3fs://bucket1.vol1/${PREFIX}-${random}
- ${result} = Execute hdfs dfs -ls o3fs://bucket1.vol1/
+ ${result} = Execute hdfs dfs -put /opt/hadoop/NOTICE.txt o3fs://bucket1.volume1/${PREFIX}-${random}
+ ${result} = Execute hdfs dfs -ls o3fs://bucket1.volume1/
Should contain ${result} ${PREFIX}-${random}
diff --git a/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot b/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot
index 8be6793..fcc7809 100644
--- a/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot
@@ -31,6 +31,10 @@
Execute ozone sh bucket create o3://om/fstest2/bucket3
Check volume from ozonefs
+ ${result} = Execute ozone sh volume list
+ Should contain ${result} fstest
+ Should contain ${result} fstest2
+ Should Match Regexp ${result} "admin" : "(hadoop|testuser\/scm@EXAMPLE\.COM)"
${result} = Execute ozone fs -ls o3fs://bucket1.fstest/
Run ozoneFS tests
diff --git a/hadoop-ozone/dist/src/main/smoketest/recon/recon-api.robot b/hadoop-ozone/dist/src/main/smoketest/recon/recon-api.robot
index 621bbd0..7073849 100644
--- a/hadoop-ozone/dist/src/main/smoketest/recon/recon-api.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/recon/recon-api.robot
@@ -62,6 +62,9 @@
Should contain ${result} \"healthyDatanodes\":3
Should contain ${result} \"pipelines\":4
+ ${result} = Execute curl --negotiate -u : -v ${API_ENDPOINT_URL}/containers/1/replicaHistory
+ Should contain ${result} \"containerId\":1
+
Check if Recon Web UI is up
Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit HTTP user
${result} = Execute curl --negotiate -u : -v ${ENDPOINT_URL}
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/bucketcreate.robot b/hadoop-ozone/dist/src/main/smoketest/s3/bucketcreate.robot
index 17762bc..56def00 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/bucketcreate.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/bucketcreate.robot
@@ -28,11 +28,11 @@
*** Test Cases ***
+Create new bucket
+ Create bucket
+
Create bucket which already exists
-# Bucket already is created in Test Setup.
- ${result} = Execute AWSS3APICli create-bucket --bucket ${BUCKET}
- Should contain ${result} ${BUCKET}
- Should contain ${result} Location
+ Create bucket with name ${BUCKET}
Create bucket with invalid bucket name
${result} = Execute AWSS3APICli and checkrc create-bucket --bucket bucket_1 255
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/bucketdelete.robot b/hadoop-ozone/dist/src/main/smoketest/s3/bucketdelete.robot
new file mode 100644
index 0000000..bcba30d
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/bucketdelete.robot
@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation S3 gateway test with aws cli
+Library OperatingSystem
+Library String
+Resource ../commonlib.robot
+Resource commonawslib.robot
+Test Timeout 5 minutes
+Suite Setup Setup s3 tests
+
+*** Variables ***
+${ENDPOINT_URL} http://s3g:9878
+${BUCKET} generated
+
+*** Test Cases ***
+
+Delete existing bucket
+# Bucket already is created in Test Setup.
+ Execute AWSS3APICli delete-bucket --bucket ${BUCKET}
+
+Delete non-existent bucket
+ ${result} = Execute AWSS3APICli and checkrc delete-bucket --bucket nosuchbucket 255
+ Should contain ${result} NoSuchBucket
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
index 9a9fd05..4595587 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
@@ -74,9 +74,17 @@
Create bucket
${postfix} = Generate Random String 5 [NUMBERS]
Set Suite Variable ${BUCKET} bucket-${postfix}
- Execute AWSS3APICli create-bucket --bucket ${BUCKET}
+ Create bucket with name ${BUCKET}
+
+Create bucket with name
+ [Arguments] ${bucket}
+ ${result} = Execute AWSS3APICli create-bucket --bucket ${bucket}
+ Should contain ${result} Location
+ Should contain ${result} ${ENDPOINT_URL}/${bucket}
Setup s3 tests
Run Keyword Install aws cli
Run Keyword if '${OZONE_S3_SET_CREDENTIALS}' == 'true' Setup v4 headers
+ ${result} = Execute And Ignore Error ozone sh volume create o3://${OM_SERVICE_ID}/s3v
+ Should not contain ${result} Failed
Run Keyword if '${BUCKET}' == 'generated' Create bucket
diff --git a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-fs.robot b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-fs.robot
index 4e36859..af9c9ae 100644
--- a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-fs.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-fs.robot
@@ -40,7 +40,7 @@
Create volume with non-admin user
Run Keyword Kinit test user testuser2 testuser2.keytab
${rc} ${output} = Run And Return Rc And Output ozone sh volume create o3://om/fstest
- Should contain ${output} Only admin users are authorized to create
+ Should contain ${output} doesn't have CREATE permission to access volume
Create volume bucket with credentials
# Authenticate testuser
diff --git a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-token.robot b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-token.robot
new file mode 100644
index 0000000..5823143
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-token.robot
@@ -0,0 +1,76 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation Test token operations
+Library OperatingSystem
+Library String
+Library BuiltIn
+Resource ../commonlib.robot
+Test Timeout 5 minutes
+
+*** Keywords ***
+Get Token in Secure Cluster
+ Execute ozone sh token get > /tmp/token.txt
+ File Should Not Be Empty /tmp/token.txt
+
+Get Token in Unsecure Cluster
+ ${output} = Execute ozone sh token get
+ Should Contain ${output} ozone sh token get
+ Should Contain ${output} only when security is enabled
+
+# should be executed after Get Token
+Print Valid Token File
+ ${output} = Execute ozone sh token print
+ Should Not Be Empty ${output}
+
+Print Nonexistent Token File
+ ${output} = Execute ozone sh token print -t /asdf
+ Should Contain ${output} operation failed as token file: /asdf
+
+Renew Token in Secure Cluster
+ ${output} = Execute ozone sh token renew
+ Should contain ${output} Token renewed successfully
+
+Renew Token in Unsecure Cluster
+ ${output} = Execute ozone sh token renew
+ Should Contain ${output} ozone sh token renew
+ Should Contain ${output} only when security is enabled
+
+Cancel Token in Secure Cluster
+ ${output} = Execute ozone sh token cancel
+ Should contain ${output} Token canceled successfully
+
+Cancel Token in Unsecure Cluster
+ ${output} = Execute ozone sh token cancel
+ Should Contain ${output} ozone sh token cancel
+ Should Contain ${output} only when security is enabled
+
+Token Test in Secure Cluster
+ Get Token in Secure Cluster
+ Print Valid Token File
+ Renew Token in Secure Cluster
+ Cancel Token in Secure Cluster
+
+Token Test in Unsecure Cluster
+ Get Token in Unsecure Cluster
+ Renew Token in Unsecure Cluster
+ Cancel Token in Unsecure Cluster
+
+*** Test Cases ***
+Token Test
+ Run Keyword if '${SECURITY_ENABLED}' == 'false' Token Test in Unsecure Cluster
+ Run Keyword if '${SECURITY_ENABLED}' == 'true' Token Test in Secure Cluster
+ Print Nonexistent Token File
diff --git a/hadoop-ozone/dist/src/main/smoketest/spnego/web.robot b/hadoop-ozone/dist/src/main/smoketest/spnego/web.robot
new file mode 100644
index 0000000..9c4156f
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/spnego/web.robot
@@ -0,0 +1,65 @@
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation Smoke test for spnego with docker-compose environments.
+Library OperatingSystem
+Library String
+Library BuiltIn
+Resource ../commonlib.robot
+Test Timeout 5 minutes
+
+*** Variables ***
+${OM_URL} http://om:9874
+${OM_DB_CHECKPOINT_URL} http://om:9874/dbCheckpoint
+${OM_SERVICE_LIST_URL} http://om:9874/serviceList
+
+${SCM_URL} http://scm:9876
+${RECON_URL} http://recon:9888
+
+*** Keywords ***
+Verify SPNEGO enabled URL
+ [arguments] ${url}
+ Run Keyword if '${SECURITY_ENABLED}' == 'true' Execute kdestroy
+ ${result} = Execute curl --negotiate -u : -v -s -I ${url}
+ Should contain ${result} 401 Unauthorized
+
+ Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user testuser testuser.keytab
+ ${result} = Execute curl --negotiate -u : -v -s -I ${url}
+ Should contain ${result} 200 OK
+
+
+
+*** Test Cases ***
+Generate Freon data
+ Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user testuser testuser.keytab
+ Execute ozone freon rk --replicationType=RATIS --numOfVolumes 1 --numOfBuckets 1 --numOfKeys 2 --keySize 1025
+
+Test OM portal
+ Verify SPNEGO enabled URL ${OM_URL}
+
+Test OM DB Checkpoint
+ Verify SPNEGO enabled URL ${OM_DB_CHECKPOINT_URL}
+
+Test OM Service List
+ Verify SPNEGO enabled URL ${OM_SERVICE_LIST_URL}
+
+Test SCM portal
+ Verify SPNEGO enabled URL ${SCM_URL}
+
+Test Recon portal
+ Verify SPNEGO enabled URL ${RECON_URL}
+
diff --git a/hadoop-ozone/dist/src/main/smoketest/topology/cli.robot b/hadoop-ozone/dist/src/main/smoketest/topology/cli.robot
index f350704..3f83ba3 100644
--- a/hadoop-ozone/dist/src/main/smoketest/topology/cli.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/topology/cli.robot
@@ -18,6 +18,7 @@
Library OperatingSystem
Library BuiltIn
Resource ../commonlib.robot
+Test Timeout 5 minutes
*** Variables ***
diff --git a/hadoop-ozone/dist/src/main/smoketest/topology/loaddata.robot b/hadoop-ozone/dist/src/main/smoketest/topology/loaddata.robot
index 7b62e11..35457e6 100644
--- a/hadoop-ozone/dist/src/main/smoketest/topology/loaddata.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/topology/loaddata.robot
@@ -18,6 +18,7 @@
Library OperatingSystem
Library BuiltIn
Resource ../commonlib.robot
+Test Timeout 5 minutes
*** Variables ***
diff --git a/hadoop-ozone/dist/src/main/smoketest/topology/readdata.robot b/hadoop-ozone/dist/src/main/smoketest/topology/readdata.robot
index 6aafef7..70c7c61 100644
--- a/hadoop-ozone/dist/src/main/smoketest/topology/readdata.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/topology/readdata.robot
@@ -18,6 +18,7 @@
Library OperatingSystem
Library BuiltIn
Resource ../commonlib.robot
+Test Timeout 5 minutes
*** Variables ***
diff --git a/hadoop-ozone/dist/src/shell/conf/ozone-shell-log4j.properties b/hadoop-ozone/dist/src/shell/conf/ozone-shell-log4j.properties
index e8f5f2d..f6935d7 100644
--- a/hadoop-ozone/dist/src/shell/conf/ozone-shell-log4j.properties
+++ b/hadoop-ozone/dist/src/shell/conf/ozone-shell-log4j.properties
@@ -31,3 +31,4 @@
log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
log4j.logger.org.apache.ratis.grpc.client.GrpcClientProtocolClient=WARN
+log4j.logger.org.apache.hadoop.io.retry.RetryInvocationHandler=WARN
diff --git a/hadoop-ozone/dist/src/shell/hdds/hadoop-functions.sh b/hadoop-ozone/dist/src/shell/hdds/hadoop-functions.sh
index 484fe23..0c45829 100755
--- a/hadoop-ozone/dist/src/shell/hdds/hadoop-functions.sh
+++ b/hadoop-ozone/dist/src/shell/hdds/hadoop-functions.sh
@@ -1541,6 +1541,21 @@
fi
}
+## @description Adds default GC parameters
+## @description Only for server components and only if no other -XX parameters
+## @description are set
+## @audience private
+## @stability evolving
+## @replaceable yes
+function hadoop_add_default_gc_opts
+{
+ if [[ "${HADOOP_SUBCMD_SUPPORTDAEMONIZATION}" == true ]]; then
+ if [[ ! "$HADOOP_OPTS" =~ "-XX" ]] ; then
+ hadoop_error "No '-XX:...' jvm parameters are used. Adding safer GC settings to the HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled"
+ fi
+ fi
+}
## @description Adds the HADOOP_CLIENT_OPTS variable to
## @description HADOOP_OPTS if HADOOP_SUBCMD_SUPPORTDAEMONIZATION is false
## @audience public
@@ -2730,3 +2745,51 @@
hadoop_java_exec "${HADOOP_SUBCMD}" "${HADOOP_CLASSNAME}" "${HADOOP_SUBCMD_ARGS[@]}"
fi
}
+
+
+## @description Add all the required jar files to the classpath
+## @audience private
+## @stability evolving
+## @replaceable yes
+function hadoop_assembly_classpath() {
+ #
+ # Setting up classpath based on the generate classpath descriptors
+ #
+ ARTIFACT_NAME="$1"
+ if [ ! "$ARTIFACT_NAME" ]; then
+ echo "ERROR: Ozone components require to set OZONE_RUN_ARTIFACT_NAME to set the classpath"
+ exit 255
+ fi
+ export HDDS_LIB_JARS_DIR="${HADOOP_HDFS_HOME}/share/ozone/lib"
+ CLASSPATH_FILE="${HADOOP_HDFS_HOME}/share/ozone/classpath/${ARTIFACT_NAME}.classpath"
+ if [ ! "$CLASSPATH_FILE" ]; then
+ echo "ERROR: Classpath file descriptor $CLASSPATH_FILE is missing"
+ exit 255
+ fi
+ # shellcheck disable=SC1090,SC2086
+ source $CLASSPATH_FILE
+ OIFS=$IFS
+ IFS=':'
+
+ # shellcheck disable=SC2154
+ for jar in $classpath; do
+ hadoop_add_classpath "$jar"
+ done
+ hadoop_add_classpath "${HADOOP_HDFS_HOME}/share/ozone/web"
+
+ #We need to add the artifact manually as it's not part the generated classpath desciptor
+ ARTIFACT_LIB_DIR="${HADOOP_HDFS_HOME}/share/ozone/lib"
+ MAIN_ARTIFACT=$(find "$ARTIFACT_LIB_DIR" -name "${OZONE_RUN_ARTIFACT_NAME}-*.jar")
+ if [ ! "$MAIN_ARTIFACT" ]; then
+ echo "ERROR: Component jar file $MAIN_ARTIFACT is missing from ${HADOOP_HDFS_HOME}/share/ozone/lib"
+ fi
+ hadoop_add_classpath "${MAIN_ARTIFACT}"
+
+ #Add optional jars to the classpath
+ OPTIONAL_CLASSPATH_DIR="${HADOOP_HDFS_HOME}/share/ozone/lib/${ARTIFACT_NAME}"
+ if [[ -d "$OPTIONAL_CLASSPATH_DIR" ]]; then
+ hadoop_add_classpath "$OPTIONAL_CLASSPATH_DIR/*"
+ fi
+
+ IFS=$OIFS
+}
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/shell/ozone/ozone b/hadoop-ozone/dist/src/shell/ozone/ozone
index ba80e07..ab7c127 100755
--- a/hadoop-ozone/dist/src/shell/ozone/ozone
+++ b/hadoop-ozone/dist/src/shell/ozone/ozone
@@ -55,7 +55,7 @@
hadoop_add_subcommand "dtutil" client "operations related to delegation tokens"
hadoop_add_subcommand "upgrade" client "HDFS to Ozone in-place upgrade tool"
hadoop_add_subcommand "admin" client "Ozone admin tool"
- hadoop_add_subcommand "ratislogparser" client "ozone debug tool, to convert Ratis log files into readable text"
+ hadoop_add_subcommand "debug" client "Ozone debug tool"
hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" false
}
@@ -130,6 +130,18 @@
OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools"
;;
genesis)
+ ARTIFACT_LIB_DIR="${HADOOP_HDFS_HOME}/share/ozone/lib/hadoop-ozone-tools"
+ mkdir -p "$ARTIFACT_LIB_DIR"
+ if [[ ! -f "$ARTIFACT_LIB_DIR/jmh-core-1.23.jar" ]]; then
+ echo "jmh-core jar is missing from $ARTIFACT_LIB_DIR, trying to download from maven central (License: GPL + classpath exception)"
+ curl -o "$ARTIFACT_LIB_DIR/jmh-core-1.23.jar" https://repo1.maven.org/maven2/org/openjdk/jmh/jmh-core/1.23/jmh-core-1.23.jar
+ fi
+
+ if [[ ! -f "$ARTIFACT_LIB_DIR/jopt-simple-4.6.jar" ]]; then
+ echo "jopt jar is missing from $ARTIFACT_LIB_DIR, trying to download from maven central (License: MIT License)"
+ curl -o "$ARTIFACT_LIB_DIR/jopt-simple-4.6.jar" https://repo1.maven.org/maven2/net/sf/jopt-simple/jopt-simple/4.6/jopt-simple-4.6.jar
+ fi
+
HADOOP_CLASSNAME=org.apache.hadoop.ozone.genesis.Genesis
OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools"
;;
@@ -145,15 +157,15 @@
OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-ozone-manager"
;;
sh | shell)
- HADOOP_CLASSNAME=org.apache.hadoop.ozone.web.ozShell.OzoneShell
+ HADOOP_CLASSNAME=org.apache.hadoop.ozone.shell.OzoneShell
HDFS_OM_SH_OPTS="${HDFS_OM_SH_OPTS} -Dhadoop.log.file=ozone-shell.log
-Dlog4j.configuration=file:${ozone_shell_log4j}"
HADOOP_OPTS="${HADOOP_OPTS} ${HDFS_OM_SH_OPTS}"
- OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-ozone-manager"
+ OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools"
;;
s3)
- HADOOP_CLASSNAME=org.apache.hadoop.ozone.web.ozShell.s3.S3Shell
- OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-ozone-manager"
+ HADOOP_CLASSNAME=org.apache.hadoop.ozone.shell.s3.S3Shell
+ OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools"
;;
scm)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
@@ -207,8 +219,8 @@
HADOOP_CLASSNAME=org.apache.hadoop.ozone.admin.OzoneAdmin
OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools"
;;
- ratislogparser)
- HADOOP_CLASSNAME=org.apache.hadoop.ozone.segmentparser.RatisLogParser
+ debug)
+ HADOOP_CLASSNAME=org.apache.hadoop.ozone.debug.OzoneDebug
OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools"
;;
*)
@@ -269,42 +281,12 @@
ozonecmd_case "${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}"
fi
-
-#
-# Setting up classpath based on the generate classpath descriptors
-#
-if [ ! "$OZONE_RUN_ARTIFACT_NAME" ]; then
- echo "ERROR: Ozone components require to set OZONE_RUN_ARTIFACT_NAME to set the classpath"
- exit -1
-fi
-export HDDS_LIB_JARS_DIR="${HADOOP_HDFS_HOME}/share/ozone/lib"
-CLASSPATH_FILE="${HADOOP_HDFS_HOME}/share/ozone/classpath/${OZONE_RUN_ARTIFACT_NAME}.classpath"
-if [ ! "$CLASSPATH_FILE" ]; then
- echo "ERROR: Classpath file descriptor $CLASSPATH_FILE is missing"
- exit -1
-fi
-# shellcheck disable=SC1090,SC2086
-source $CLASSPATH_FILE
-OIFS=$IFS
-IFS=':'
-# shellcheck disable=SC2154
-for jar in $classpath; do
- hadoop_add_classpath "$jar"
-done
-hadoop_add_classpath "${HADOOP_HDFS_HOME}/share/ozone/web"
-
-#We need to add the artifact manually as it's not part the generated classpath desciptor
-ARTIFACT_LIB_DIR="${HADOOP_HDFS_HOME}/share/ozone/lib"
-MAIN_ARTIFACT=$(find "$ARTIFACT_LIB_DIR" -name "${OZONE_RUN_ARTIFACT_NAME}-*.jar")
-if [ ! "$MAIN_ARTIFACT" ]; then
- echo "ERROR: Component jar file $MAIN_ARTIFACT is missing from ${HADOOP_HDFS_HOME}/share/ozone/lib"
-fi
-hadoop_add_classpath "${MAIN_ARTIFACT}"
-IFS=$OIFS
-
+hadoop_assembly_classpath "$OZONE_RUN_ARTIFACT_NAME"
hadoop_add_client_opts
+hadoop_add_default_gc_opts
+
if [[ ${HADOOP_WORKER_MODE} = true ]]; then
hadoop_common_worker_mode_execute "${HADOOP_HDFS_HOME}/bin/ozone" "${HADOOP_USER_PARAMS[@]}"
exit $?
diff --git a/hadoop-ozone/dist/src/test/shell/gc_opts.bats b/hadoop-ozone/dist/src/test/shell/gc_opts.bats
new file mode 100644
index 0000000..1400a40
--- /dev/null
+++ b/hadoop-ozone/dist/src/test/shell/gc_opts.bats
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+
+#
+# Can be executed with bats (https://github.com/bats-core/bats-core)
+# bats gc_opts.bats (FROM THE CURRENT DIRECTORY)
+#
+
+source ../../shell/hdds/hadoop-functions.sh
+@test "Setting Hadoop GC parameters: add GC params for server" {
+ export HADOOP_SUBCMD_SUPPORTDAEMONIZATION=true
+ export HADOOP_OPTS="Test"
+ hadoop_add_default_gc_opts
+ [[ "$HADOOP_OPTS" =~ "UseConcMarkSweepGC" ]]
+}
+
+@test "Setting Hadoop GC parameters: disabled for client" {
+ export HADOOP_SUBCMD_SUPPORTDAEMONIZATION=false
+ export HADOOP_OPTS="Test"
+ hadoop_add_default_gc_opts
+ [[ ! "$HADOOP_OPTS" =~ "UseConcMarkSweepGC" ]]
+}
+
+@test "Setting Hadoop GC parameters: disabled if GC params are customized" {
+ export HADOOP_SUBCMD_SUPPORTDAEMONIZATION=true
+ export HADOOP_OPTS="-XX:++UseG1GC -Xmx512"
+ hadoop_add_default_gc_opts
+ [[ ! "$HADOOP_OPTS" =~ "UseConcMarkSweepGC" ]]
+}
diff --git a/hadoop-ozone/fault-injection-test/mini-chaos-tests/pom.xml b/hadoop-ozone/fault-injection-test/mini-chaos-tests/pom.xml
index 3bb6419..941a9cf 100644
--- a/hadoop-ozone/fault-injection-test/mini-chaos-tests/pom.xml
+++ b/hadoop-ozone/fault-injection-test/mini-chaos-tests/pom.xml
@@ -35,6 +35,11 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-test-utils</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-filesystem</artifactId>
</dependency>
<dependency>
@@ -45,9 +50,8 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
+ <artifactId>hadoop-hdds-hadoop-dependency-test</artifactId>
<scope>test</scope>
- <type>test-jar</type>
</dependency>
</dependencies>
-</project>
\ No newline at end of file
+</project>
diff --git a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java
index 65eb86d..dc7c26c 100644
--- a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java
+++ b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java
@@ -19,33 +19,37 @@
package org.apache.hadoop.ozone;
import java.util.Arrays;
-import java.util.concurrent.ExecutionException;
+import java.util.List;
+import java.util.HashSet;
+import java.util.ArrayList;
+import java.util.Set;
+import java.util.Collections;
import java.util.concurrent.TimeoutException;
import org.apache.commons.lang3.RandomUtils;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.ozone.failure.FailureManager;
+import org.apache.hadoop.ozone.failure.Failures;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.test.GenericTestUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.util.List;
-
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.Executors;
/**
* This class causes random failures in the chaos cluster.
*/
-public abstract class MiniOzoneChaosCluster extends MiniOzoneHAClusterImpl {
+public class MiniOzoneChaosCluster extends MiniOzoneHAClusterImpl {
static final Logger LOG =
LoggerFactory.getLogger(MiniOzoneChaosCluster.class);
@@ -53,21 +57,12 @@
private final int numDatanodes;
private final int numOzoneManagers;
- // Number of Nodes of the service (Datanode or OM) on which chaos will be
- // unleashed
- private int numNodes;
+ private final FailureManager failureManager;
- private FailureService failureService;
- private long failureIntervalInMS;
+ private final int waitForClusterToBeReadyTimeout = 120000; // 2 min
- private final ScheduledExecutorService executorService;
-
- private ScheduledFuture scheduledFuture;
-
- private enum FailureMode {
- NODES_RESTART,
- NODES_SHUTDOWN
- }
+ private final Set<OzoneManager> failedOmSet;
+ private final Set<DatanodeDetails> failedDnSet;
// The service on which chaos will be unleashed.
enum FailureService {
@@ -96,166 +91,30 @@
public MiniOzoneChaosCluster(OzoneConfiguration conf,
List<OzoneManager> ozoneManagers, StorageContainerManager scm,
List<HddsDatanodeService> hddsDatanodes, String omServiceID,
- FailureService service) {
+ List<Class<? extends Failures>> clazzes) {
super(conf, ozoneManagers, scm, hddsDatanodes, omServiceID);
-
- this.executorService = Executors.newSingleThreadScheduledExecutor();
this.numDatanodes = getHddsDatanodes().size();
this.numOzoneManagers = ozoneManagers.size();
- this.failureService = service;
+
+ this.failedOmSet = new HashSet<>();
+ this.failedDnSet = new HashSet<>();
+
+ this.failureManager = new FailureManager(this, conf, clazzes);
LOG.info("Starting MiniOzoneChaosCluster with {} OzoneManagers and {} " +
- "Datanodes, chaos on service: {}",
- numOzoneManagers, numDatanodes, failureService);
- }
-
- protected int getNumNodes() {
- return numNodes;
- }
-
- protected void setNumNodes(int numOfNodes) {
- this.numNodes = numOfNodes;
- }
-
- protected long getFailureIntervalInMS() {
- return failureIntervalInMS;
- }
-
- /**
- * Is the cluster ready for chaos.
- */
- protected boolean isClusterReady() {
- return true;
- }
-
- protected void getClusterReady() {
- // Do nothing
- }
-
- // Get the number of nodes to fail in the cluster.
- protected int getNumberOfNodesToFail() {
- return RandomUtils.nextBoolean() ? 1 : 2;
- }
-
- // Should the failed node wait for SCM to register even before
- // restart, i.e fast restart or not.
- protected boolean isFastRestart() {
- return RandomUtils.nextBoolean();
- }
-
- // Should the selected node be stopped or started.
- protected boolean shouldStop() {
- return RandomUtils.nextBoolean();
- }
-
- // Get the node index of the node to fail.
- private int getNodeToFail() {
- return RandomUtils.nextInt() % numNodes;
- }
-
- protected abstract void restartNode(int failedNodeIndex,
- boolean waitForNodeRestart)
- throws TimeoutException, InterruptedException, IOException;
-
- protected abstract void shutdownNode(int failedNodeIndex)
- throws ExecutionException, InterruptedException;
-
- protected abstract String getFailedNodeID(int failedNodeIndex);
-
- private void restartNodes() {
- final int numNodesToFail = getNumberOfNodesToFail();
- LOG.info("Will restart {} nodes to simulate failure", numNodesToFail);
- for (int i = 0; i < numNodesToFail; i++) {
- boolean failureMode = isFastRestart();
- int failedNodeIndex = getNodeToFail();
- String failString = failureMode ? "Fast" : "Slow";
- String failedNodeID = getFailedNodeID(failedNodeIndex);
- try {
- LOG.info("{} Restarting {}: {}", failString, failureService,
- failedNodeID);
- restartNode(failedNodeIndex, failureMode);
- LOG.info("{} Completed restarting {}: {}", failString, failureService,
- failedNodeID);
- } catch (Exception e) {
- LOG.error("Failed to restartNodes {}: {}", failedNodeID,
- failureService, e);
- }
- }
- }
-
- private void shutdownNodes() {
- final int numNodesToFail = getNumberOfNodesToFail();
- LOG.info("Will shutdown {} nodes to simulate failure", numNodesToFail);
- for (int i = 0; i < numNodesToFail; i++) {
- boolean shouldStop = shouldStop();
- int failedNodeIndex = getNodeToFail();
- String stopString = shouldStop ? "Stopping" : "Restarting";
- String failedNodeID = getFailedNodeID(failedNodeIndex);
- try {
- LOG.info("{} {} {}", stopString, failureService, failedNodeID);
- if (shouldStop) {
- shutdownNode(failedNodeIndex);
- } else {
- restartNode(failedNodeIndex, false);
- }
- LOG.info("Completed {} {} {}", stopString, failureService,
- failedNodeID);
- } catch (Exception e) {
- LOG.error("Failed {} {} {}", stopString, failureService,
- failedNodeID, e);
- }
- }
- }
-
- private FailureMode getFailureMode() {
- return FailureMode.
- values()[RandomUtils.nextInt() % FailureMode.values().length];
- }
-
- // Fail nodes randomly at configured timeout period.
- private void fail() {
- if (isClusterReady()) {
- FailureMode mode = getFailureMode();
- switch (mode) {
- case NODES_RESTART:
- restartNodes();
- break;
- case NODES_SHUTDOWN:
- shutdownNodes();
- break;
-
- default:
- LOG.error("invalid failure mode:{}", mode);
- break;
- }
- } else {
- // Cluster is not ready for failure yet. Skip failing this time and get
- // the cluster ready by restarting any OM that is not running.
- LOG.info("Cluster is not ready for failure.");
- getClusterReady();
- }
+ "Datanodes", numOzoneManagers, numDatanodes);
+ clazzes.forEach(c -> LOG.info("added failure:{}", c.getSimpleName()));
}
void startChaos(long initialDelay, long period, TimeUnit timeUnit) {
LOG.info("Starting Chaos with failure period:{} unit:{} numDataNodes:{} " +
"numOzoneManagers:{}", period, timeUnit, numDatanodes,
numOzoneManagers);
- this.failureIntervalInMS = TimeUnit.MILLISECONDS.convert(period, timeUnit);
- scheduledFuture = executorService.scheduleAtFixedRate(this::fail,
- initialDelay, period, timeUnit);
- }
-
- void stopChaos() throws Exception {
- if (scheduledFuture != null) {
- scheduledFuture.cancel(false);
- scheduledFuture.get();
- }
+ failureManager.start(initialDelay, period, timeUnit);
}
public void shutdown() {
try {
- stopChaos();
- executorService.shutdown();
- executorService.awaitTermination(1, TimeUnit.DAYS);
+ failureManager.stop();
//this should be called after stopChaos to be sure that the
//datanode collection is not modified during the shutdown
super.shutdown();
@@ -265,11 +124,30 @@
}
/**
+ * Check if cluster is ready for a restart or shutdown of an OM node. If
+ * yes, then set isClusterReady to false so that another thread cannot
+ * restart/ shutdown OM till all OMs are up again.
+ */
+ @Override
+ public void waitForClusterToBeReady()
+ throws TimeoutException, InterruptedException {
+ super.waitForClusterToBeReady();
+ GenericTestUtils.waitFor(() -> {
+ for (OzoneManager om : getOzoneManagersList()) {
+ if (!om.isRunning()) {
+ return false;
+ }
+ }
+ return true;
+ }, 1000, waitForClusterToBeReadyTimeout);
+ }
+
+ /**
* Builder for configuring the MiniOzoneChaosCluster to run.
*/
public static class Builder extends MiniOzoneHAClusterImpl.Builder {
- private FailureService failureService;
+ private final List<Class<? extends Failures>> clazzes = new ArrayList<>();
/**
* Creates a new Builder.
@@ -311,8 +189,8 @@
return this;
}
- public Builder setFailureService(String serviceName) {
- this.failureService = FailureService.of(serviceName);
+ public Builder addFailures(Class<? extends Failures> clazz) {
+ this.clazzes.add(clazz);
return this;
}
@@ -358,17 +236,27 @@
conf.setInt("hdds.scm.replication.event.timeout", 20 * 1000);
conf.setInt(OzoneConfigKeys.DFS_RATIS_SNAPSHOT_THRESHOLD_KEY, 100);
conf.setInt(OzoneConfigKeys.DFS_CONTAINER_RATIS_LOG_PURGE_GAP, 100);
+
+ conf.setInt(OMConfigKeys.
+ OZONE_OM_RATIS_SNAPSHOT_AUTO_TRIGGER_THRESHOLD_KEY, 100);
+ conf.setInt(OMConfigKeys.OZONE_OM_RATIS_LOG_PURGE_GAP, 100);
+ }
+
+ /**
+ * Sets the number of data volumes per datanode.
+ *
+ * @param val number of volumes per datanode.
+ *
+ * @return MiniOzoneCluster.Builder
+ */
+ public Builder setNumDataVolumes(int val) {
+ numDataVolumes = val;
+ return this;
}
@Override
public MiniOzoneChaosCluster build() throws IOException {
- if (failureService == FailureService.OZONE_MANAGER && numOfOMs < 3) {
- throw new IllegalArgumentException("Not enough number of " +
- "OzoneManagers to test chaos on OzoneManagers. Set number of " +
- "OzoneManagers to at least 3");
- }
-
DefaultMetricsSystem.setMiniClusterMode(true);
initializeConfiguration();
if (numOfOMs > 1) {
@@ -394,14 +282,9 @@
final List<HddsDatanodeService> hddsDatanodes = createHddsDatanodes(
scm, null);
- MiniOzoneChaosCluster cluster;
- if (failureService == FailureService.DATANODE) {
- cluster = new MiniOzoneDatanodeChaosCluster(conf, omList, scm,
- hddsDatanodes, omServiceId);
- } else {
- cluster = new MiniOzoneOMChaosCluster(conf, omList, scm,
- hddsDatanodes, omServiceId);
- }
+ MiniOzoneChaosCluster cluster =
+ new MiniOzoneChaosCluster(conf, omList, scm, hddsDatanodes,
+ omServiceId, clazzes);
if (startDataNodes) {
cluster.startHddsDatanodes();
@@ -409,4 +292,78 @@
return cluster;
}
}
+
+ // OzoneManager specifc
+ public static int getNumberOfOmToFail() {
+ return 1;
+ }
+
+ public Set<OzoneManager> omToFail() {
+ int numNodesToFail = getNumberOfOmToFail();
+ if (failedOmSet.size() >= numOzoneManagers/2) {
+ return Collections.emptySet();
+ }
+
+ int numOms = getOzoneManagersList().size();
+ Set<OzoneManager> oms = new HashSet<>();
+ for (int i = 0; i < numNodesToFail; i++) {
+ int failedNodeIndex = FailureManager.getBoundedRandomIndex(numOms);
+ oms.add(getOzoneManager(failedNodeIndex));
+ }
+ return oms;
+ }
+
+ public void shutdownOzoneManager(OzoneManager om) {
+ super.shutdownOzoneManager(om);
+ failedOmSet.add(om);
+ }
+
+ public void restartOzoneManager(OzoneManager om, boolean waitForOM)
+ throws IOException, TimeoutException, InterruptedException {
+ super.restartOzoneManager(om, waitForOM);
+ failedOmSet.remove(om);
+ }
+
+ // Should the selected node be stopped or started.
+ public boolean shouldStop() {
+ if (failedOmSet.size() >= numOzoneManagers/2) {
+ return false;
+ }
+ return RandomUtils.nextBoolean();
+ }
+
+ // Datanode specifc
+ private int getNumberOfDnToFail() {
+ return RandomUtils.nextBoolean() ? 1 : 2;
+ }
+
+ public Set<DatanodeDetails> dnToFail() {
+ int numNodesToFail = getNumberOfDnToFail();
+ int numDns = getHddsDatanodes().size();
+ Set<DatanodeDetails> dns = new HashSet<>();
+ for (int i = 0; i < numNodesToFail; i++) {
+ int failedNodeIndex = FailureManager.getBoundedRandomIndex(numDns);
+ dns.add(getHddsDatanodes().get(failedNodeIndex).getDatanodeDetails());
+ }
+ return dns;
+ }
+
+ @Override
+ public void restartHddsDatanode(DatanodeDetails dn, boolean waitForDatanode)
+ throws InterruptedException, TimeoutException, IOException {
+ failedDnSet.add(dn);
+ super.restartHddsDatanode(dn, waitForDatanode);
+ failedDnSet.remove(dn);
+ }
+
+ @Override
+ public void shutdownHddsDatanode(DatanodeDetails dn) throws IOException {
+ failedDnSet.add(dn);
+ super.shutdownHddsDatanode(dn);
+ }
+
+ // Should the selected node be stopped or started.
+ public boolean shouldStop(DatanodeDetails dn) {
+ return !failedDnSet.contains(dn);
+ }
}
diff --git a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneDatanodeChaosCluster.java b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneDatanodeChaosCluster.java
deleted file mode 100644
index f402831..0000000
--- a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneDatanodeChaosCluster.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone;
-
-import java.util.List;
-import java.util.concurrent.TimeoutException;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
-import org.apache.hadoop.ozone.om.OzoneManager;
-
-/**
- * This class causes random failures in Datanodes in the chaos cluster.
- */
-public class MiniOzoneDatanodeChaosCluster extends MiniOzoneChaosCluster {
-
- public MiniOzoneDatanodeChaosCluster(OzoneConfiguration conf,
- List<OzoneManager> ozoneManagers,
- StorageContainerManager scm,
- List<HddsDatanodeService> hddsDatanodes,
- String omServiceID) {
- super(conf, ozoneManagers, scm, hddsDatanodes, omServiceID,
- FailureService.DATANODE);
- setNumNodes(hddsDatanodes.size());
- }
-
- @Override
- protected void restartNode(int failedNodeIndex, boolean waitForNodeRestart)
- throws TimeoutException, InterruptedException {
- restartHddsDatanode(failedNodeIndex, waitForNodeRestart);
- }
-
- @Override
- protected void shutdownNode(int failedNodeIndex) {
- shutdownHddsDatanode(failedNodeIndex);
- }
-
- @Override
- protected String getFailedNodeID(int failedNodeIndex) {
- return getHddsDatanodes().get(failedNodeIndex).getDatanodeDetails()
- .getUuidString();
- }
-}
diff --git a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneLoadGenerator.java b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneLoadGenerator.java
index 62a1db2..b7549ca 100644
--- a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneLoadGenerator.java
+++ b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneLoadGenerator.java
@@ -20,9 +20,6 @@
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.loadgenerators.FilesystemLoadGenerator;
-import org.apache.hadoop.ozone.loadgenerators.AgedLoadGenerator;
-import org.apache.hadoop.ozone.loadgenerators.RandomLoadGenerator;
import org.apache.hadoop.ozone.loadgenerators.DataBuffer;
import org.apache.hadoop.ozone.loadgenerators.LoadExecutors;
import org.apache.hadoop.ozone.loadgenerators.LoadGenerator;
@@ -32,7 +29,6 @@
import java.util.ArrayList;
import java.util.List;
-import java.util.function.Function;
import java.util.concurrent.TimeUnit;
/**
@@ -43,62 +39,99 @@
private static final Logger LOG =
LoggerFactory.getLogger(MiniOzoneLoadGenerator.class);
- private final List<LoadExecutors> loadExecutors;
+ private final List<LoadGenerator> loadGenerators;
+ private final LoadExecutors loadExecutor;
private final OzoneVolume volume;
private final OzoneConfiguration conf;
private final String omServiceID;
- MiniOzoneLoadGenerator(OzoneVolume volume, int numClients, int numThreads,
- int numBuffers, OzoneConfiguration conf, String omServiceId)
+ MiniOzoneLoadGenerator(OzoneVolume volume, int numThreads,
+ int numBuffers, OzoneConfiguration conf, String omServiceId,
+ List<Class<? extends LoadGenerator>> loadGenratorClazzes)
throws Exception {
DataBuffer buffer = new DataBuffer(numBuffers);
- loadExecutors = new ArrayList<>();
+ loadGenerators = new ArrayList<>();
this.volume = volume;
this.conf = conf;
this.omServiceID = omServiceId;
- // Random Load
- String mixBucketName = RandomStringUtils.randomAlphabetic(10).toLowerCase();
- volume.createBucket(mixBucketName);
- List<LoadBucket> ozoneBuckets = new ArrayList<>(numClients);
- for (int i = 0; i < numClients; i++) {
- ozoneBuckets.add(new LoadBucket(volume.getBucket(mixBucketName),
- conf, omServiceId));
+ for(Class<? extends LoadGenerator> clazz : loadGenratorClazzes) {
+ addLoads(clazz, buffer);
}
- RandomLoadGenerator loadGenerator =
- new RandomLoadGenerator(buffer, ozoneBuckets);
- loadExecutors.add(new LoadExecutors(numThreads, loadGenerator));
- // Aged Load
- addLoads(numThreads,
- bucket -> new AgedLoadGenerator(buffer, bucket));
-
- //Filesystem Load
- addLoads(numThreads,
- bucket -> new FilesystemLoadGenerator(buffer, bucket));
+ this.loadExecutor = new LoadExecutors(numThreads, loadGenerators);
}
- private void addLoads(int numThreads,
- Function<LoadBucket, LoadGenerator> function)
- throws Exception {
+ private void addLoads(Class<? extends LoadGenerator> clazz,
+ DataBuffer buffer) throws Exception {
String bucketName = RandomStringUtils.randomAlphabetic(10).toLowerCase();
volume.createBucket(bucketName);
- LoadBucket bucket = new LoadBucket(volume.getBucket(bucketName), conf,
- omServiceID);
- LoadGenerator loadGenerator = function.apply(bucket);
- loadExecutors.add(new LoadExecutors(numThreads, loadGenerator));
+ LoadBucket ozoneBucket = new LoadBucket(volume.getBucket(bucketName),
+ conf, omServiceID);
+
+ LoadGenerator loadGenerator = clazz
+ .getConstructor(DataBuffer.class, LoadBucket.class)
+ .newInstance(buffer, ozoneBucket);
+ loadGenerators.add(loadGenerator);
}
- void startIO(long time, TimeUnit timeUnit) {
+ void startIO(long time, TimeUnit timeUnit) throws Exception {
LOG.info("Starting MiniOzoneLoadGenerator for time {}:{}", time, timeUnit);
long runTime = timeUnit.toMillis(time);
// start and wait for executors to finish
- loadExecutors.forEach(le -> le.startLoad(runTime));
- loadExecutors.forEach(LoadExecutors::waitForCompletion);
+ loadExecutor.startLoad(runTime);
+ loadExecutor.waitForCompletion();
}
void shutdownLoadGenerator() {
- loadExecutors.forEach(LoadExecutors::shutdown);
+ loadExecutor.shutdown();
+ }
+
+ /**
+ * Builder to create Ozone load generator.
+ */
+ public static class Builder {
+ private List<Class<? extends LoadGenerator>> clazzes = new ArrayList<>();
+ private String omServiceId;
+ private OzoneConfiguration conf;
+ private int numBuffers;
+ private int numThreads;
+ private OzoneVolume volume;
+
+ public Builder addLoadGenerator(Class<? extends LoadGenerator> clazz) {
+ clazzes.add(clazz);
+ return this;
+ }
+
+ public Builder setOMServiceId(String serviceId) {
+ omServiceId = serviceId;
+ return this;
+ }
+
+ public Builder setConf(OzoneConfiguration configuration) {
+ this.conf = configuration;
+ return this;
+ }
+
+ public Builder setNumBuffers(int buffers) {
+ this.numBuffers = buffers;
+ return this;
+ }
+
+ public Builder setNumThreads(int threads) {
+ this.numThreads = threads;
+ return this;
+ }
+
+ public Builder setVolume(OzoneVolume vol) {
+ this.volume = vol;
+ return this;
+ }
+
+ public MiniOzoneLoadGenerator build() throws Exception {
+ return new MiniOzoneLoadGenerator(volume, numThreads, numBuffers,
+ conf, omServiceId, clazzes);
+ }
}
}
diff --git a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneOMChaosCluster.java b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneOMChaosCluster.java
deleted file mode 100644
index 2b2a4d7..0000000
--- a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneOMChaosCluster.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
-import org.apache.hadoop.ozone.om.OzoneManager;
-
-/**
- * This class causes random failures in OMs in the chaos cluster.
- */
-public class MiniOzoneOMChaosCluster extends MiniOzoneChaosCluster {
-
- // Cluster is deemed ready for chaos when all the OMs are up and running.
- private AtomicBoolean isClusterReady = new AtomicBoolean(true);
-
- // The maximum number of nodes failures which can be tolerated without
- // losing quorum. This should be equal to (Num of OMs - 1)/2.
- private int numOfOMNodeFailuresTolerated;
-
- MiniOzoneOMChaosCluster(OzoneConfiguration conf,
- List<OzoneManager> ozoneManagers,
- StorageContainerManager scm,
- List<HddsDatanodeService> hddsDatanodes,
- String omServiceID) {
- super(conf, ozoneManagers, scm, hddsDatanodes, omServiceID,
- FailureService.OZONE_MANAGER);
- setNumNodes(ozoneManagers.size());
- numOfOMNodeFailuresTolerated = (getNumNodes() - 1) / 2;
- }
-
- /**
- * Check if cluster is ready for a restart or shutdown of an OM node. If
- * yes, then set isClusterReady to false so that another thread cannot
- * restart/ shutdown OM till all OMs are up again.
- */
- protected boolean isClusterReady() {
- return isClusterReady.compareAndSet(true, false);
- }
-
- /**
- * If any OM node is not running, restart it.
- */
- @Override
- protected void getClusterReady() {
- boolean clusterReady = true;
- for (OzoneManager om : getOzoneManagersList()) {
- if (!om.isRunning()) {
- try {
- restartOzoneManager(om, true);
- } catch (Exception e) {
- clusterReady = false;
- LOG.error("Cluster not ready for chaos. Failed to restart OM {}: {}",
- om.getOMNodeId(), e);
- }
- }
- }
- if (clusterReady) {
- isClusterReady.set(true);
- }
- }
-
- @Override
- protected int getNumberOfNodesToFail() {
- return RandomUtils.nextInt(1, numOfOMNodeFailuresTolerated + 1);
- }
-
- @Override
- protected void restartNode(int failedNodeIndex, boolean waitForNodeRestart)
- throws IOException, TimeoutException, InterruptedException {
- shutdownOzoneManager(failedNodeIndex);
- restartOzoneManager(failedNodeIndex, waitForNodeRestart);
- getClusterReady();
- }
-
- /**
- * For OM chaos, a shutdown node should eventually be restarted before the
- * next failure.
- */
- @Override
- protected void shutdownNode(int failedNodeIndex)
- throws ExecutionException, InterruptedException {
- shutdownOzoneManager(failedNodeIndex);
-
- // Restart the OM after FailureInterval / 2 duration.
- Executors.newSingleThreadScheduledExecutor().schedule(
- this::getClusterReady, getFailureIntervalInMS() / 2,
- TimeUnit.MILLISECONDS).get();
- }
-
- @Override
- protected String getFailedNodeID(int failedNodeIndex) {
- return getOzoneManager(failedNodeIndex).getOMNodeId();
- }
-
- /**
- * When restarting OM, always wait for it to catch up with Leader OM.
- */
- @Override
- protected boolean isFastRestart() {
- return true;
- }
-
- @Override
- protected boolean shouldStop() {
- return true;
- }
-}
diff --git a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java
index 53718e4..9ad2a16 100644
--- a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java
+++ b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java
@@ -18,17 +18,27 @@
package org.apache.hadoop.ozone;
import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.hdds.cli.GenericCli;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.MiniOzoneChaosCluster.FailureService;
+import org.apache.hadoop.ozone.failure.Failures;
+import org.apache.hadoop.ozone.loadgenerators.RandomLoadGenerator;
+import org.apache.hadoop.ozone.loadgenerators.ReadOnlyLoadGenerator;
+import org.apache.hadoop.ozone.loadgenerators.FilesystemLoadGenerator;
+import org.apache.hadoop.ozone.loadgenerators.AgedLoadGenerator;
+import org.apache.hadoop.ozone.loadgenerators.AgedDirLoadGenerator;
+import org.apache.hadoop.ozone.loadgenerators.RandomDirLoadGenerator;
+import org.apache.hadoop.ozone.loadgenerators.NestedDirLoadGenerator;
import org.junit.BeforeClass;
import org.junit.AfterClass;
import org.junit.Ignore;
import org.junit.Test;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
-import picocli.CommandLine;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.concurrent.TimeUnit;
@@ -38,7 +48,9 @@
@Ignore
@Command(description = "Starts IO with MiniOzoneChaosCluster",
name = "chaos", mixinStandardHelpOptions = true)
-public class TestMiniChaosOzoneCluster implements Runnable {
+public class TestMiniChaosOzoneCluster extends GenericCli {
+ static final Logger LOG =
+ LoggerFactory.getLogger(TestMiniChaosOzoneCluster.class);
@Option(names = {"-d", "--numDatanodes"},
description = "num of datanodes")
@@ -65,9 +77,9 @@
description = "total run time")
private static int numMinutes = 1440; // 1 day by default
- @Option(names = {"-n", "--numClients"},
- description = "no of clients writing to OM")
- private static int numClients = 3;
+ @Option(names = {"-v", "--numDataVolume"},
+ description = "number of datanode volumes to create")
+ private static int numDataVolumes = 3;
@Option(names = {"-i", "--failureInterval"},
description = "time between failure events in seconds")
@@ -81,16 +93,36 @@
@BeforeClass
public static void init() throws Exception {
OzoneConfiguration configuration = new OzoneConfiguration();
- String omServiceID =
- FailureService.of(failureService) == FailureService.OZONE_MANAGER ?
- OM_SERVICE_ID : null;
+ FailureService service = FailureService.of(failureService);
+ String omServiceID;
- cluster = new MiniOzoneChaosCluster.Builder(configuration)
+ MiniOzoneChaosCluster.Builder builder =
+ new MiniOzoneChaosCluster.Builder(configuration);
+
+ switch (service) {
+ case DATANODE:
+ omServiceID = null;
+ builder
+ .addFailures(Failures.DatanodeRestartFailure.class)
+ .addFailures(Failures.DatanodeStartStopFailure.class);
+ break;
+ case OZONE_MANAGER:
+ omServiceID = OM_SERVICE_ID;
+ builder
+ .addFailures(Failures.OzoneManagerStartStopFailure.class)
+ .addFailures(Failures.OzoneManagerRestartFailure.class);
+ break;
+ default:
+ throw new IllegalArgumentException();
+ }
+
+ builder
.setNumDatanodes(numDatanodes)
.setNumOzoneManagers(numOzoneManagers)
- .setFailureService(failureService)
.setOMServiceID(omServiceID)
- .build();
+ .setNumDataVolumes(numDataVolumes);
+
+ cluster = builder.build();
cluster.waitForClusterToBeReady();
String volumeName = RandomStringUtils.randomAlphabetic(10).toLowerCase();
@@ -98,9 +130,20 @@
store.createVolume(volumeName);
OzoneVolume volume = store.getVolume(volumeName);
- loadGenerator =
- new MiniOzoneLoadGenerator(volume, numClients, numThreads,
- numBuffers, configuration, omServiceID);
+ loadGenerator = new MiniOzoneLoadGenerator.Builder()
+ .setVolume(volume)
+ .setConf(configuration)
+ .setNumBuffers(numBuffers)
+ .setNumThreads(numThreads)
+ .setOMServiceId(omServiceID)
+ .addLoadGenerator(RandomLoadGenerator.class)
+ .addLoadGenerator(AgedLoadGenerator.class)
+ .addLoadGenerator(FilesystemLoadGenerator.class)
+ .addLoadGenerator(ReadOnlyLoadGenerator.class)
+ .addLoadGenerator(RandomDirLoadGenerator.class)
+ .addLoadGenerator(AgedDirLoadGenerator.class)
+ .addLoadGenerator(NestedDirLoadGenerator.class)
+ .build();
}
/**
@@ -117,23 +160,24 @@
}
}
- public void run() {
+ @Override
+ public Void call() throws Exception {
try {
init();
cluster.startChaos(failureInterval, failureInterval, TimeUnit.SECONDS);
loadGenerator.startIO(numMinutes, TimeUnit.MINUTES);
- } catch (Exception e) {
} finally {
shutdown();
}
+ return null;
}
public static void main(String... args) {
- CommandLine.run(new TestMiniChaosOzoneCluster(), System.err, args);
+ new TestMiniChaosOzoneCluster().run(args);
}
@Test
- public void testReadWriteWithChaosCluster() {
+ public void testReadWriteWithChaosCluster() throws Exception {
cluster.startChaos(5, 10, TimeUnit.SECONDS);
loadGenerator.startIO(120, TimeUnit.SECONDS);
}
diff --git a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/failure/FailureManager.java b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/failure/FailureManager.java
new file mode 100644
index 0000000..15aa7f0
--- /dev/null
+++ b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/failure/FailureManager.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.failure;
+
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ozone.MiniOzoneChaosCluster;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Manages all the failures in the MiniOzoneChaosCluster.
+ */
+public class FailureManager {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(Failures.class);
+
+ private final MiniOzoneChaosCluster cluster;
+ private final List<Failures> failures;
+ private ScheduledFuture scheduledFuture;
+ private final ScheduledExecutorService executorService;
+ public FailureManager(MiniOzoneChaosCluster cluster,
+ Configuration conf,
+ List<Class<? extends Failures>> clazzes) {
+ this.cluster = cluster;
+ this.executorService = Executors.newSingleThreadScheduledExecutor();
+
+ failures = new ArrayList<>();
+ for (Class<? extends Failures> clazz : clazzes) {
+ Failures f = ReflectionUtils.newInstance(clazz, conf);
+ f.validateFailure(cluster);
+ failures.add(f);
+ }
+
+ }
+
+ // Fail nodes randomly at configured timeout period.
+ private void fail() {
+ Failures f = failures.get(getBoundedRandomIndex(failures.size()));
+ try {
+ LOG.info("time failure with {}", f.getName());
+ f.fail(cluster);
+ } catch (Throwable t) {
+ LOG.info("Caught exception while inducing failure:{}", f.getName(), t);
+ System.exit(-2);
+ }
+
+ }
+
+ public void start(long initialDelay, long period, TimeUnit timeUnit) {
+ LOG.info("starting failure manager {} {} {}", initialDelay,
+ period, timeUnit);
+ scheduledFuture = executorService.scheduleAtFixedRate(this::fail,
+ initialDelay, period, timeUnit);
+ }
+
+ public void stop() throws Exception {
+ if (scheduledFuture != null) {
+ scheduledFuture.cancel(false);
+ scheduledFuture.get();
+ }
+
+ executorService.shutdown();
+ executorService.awaitTermination(1, TimeUnit.MINUTES);
+ }
+
+ public static boolean isFastRestart() {
+ return RandomUtils.nextBoolean();
+ }
+
+ public static int getBoundedRandomIndex(int size) {
+ return RandomUtils.nextInt(0, size);
+ }
+}
diff --git a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/failure/Failures.java b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/failure/Failures.java
new file mode 100644
index 0000000..6d226ca
--- /dev/null
+++ b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/failure/Failures.java
@@ -0,0 +1,147 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.failure;
+
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.ozone.MiniOzoneChaosCluster;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Set;
+
+/**
+ * Implementation of all the failures.
+ */
+public abstract class Failures {
+ static final Logger LOG =
+ LoggerFactory.getLogger(Failures.class);
+
+ public String getName() {
+ return this.getClass().getSimpleName();
+ }
+
+ public abstract void fail(MiniOzoneChaosCluster cluster);
+
+ public abstract void validateFailure(MiniOzoneChaosCluster cluster);
+
+ /**
+ * Ozone Manager failures.
+ */
+ public abstract static class OzoneFailures extends Failures {
+ @Override
+ public void validateFailure(MiniOzoneChaosCluster cluster) {
+ if (cluster.getOzoneManagersList().size() < 3) {
+ throw new IllegalArgumentException("Not enough number of " +
+ "OzoneManagers to test chaos on OzoneManagers. Set number of " +
+ "OzoneManagers to at least 3");
+ }
+ }
+ }
+
+ /**
+ * Restart Ozone Manager to induce failure.
+ */
+ public static class OzoneManagerRestartFailure extends OzoneFailures {
+ public void fail(MiniOzoneChaosCluster cluster) {
+ boolean failureMode = FailureManager.isFastRestart();
+ Set<OzoneManager> oms = cluster.omToFail();
+ oms.parallelStream().forEach(om -> {
+ try {
+ cluster.shutdownOzoneManager(om);
+ cluster.restartOzoneManager(om, failureMode);
+ cluster.waitForClusterToBeReady();
+ } catch (Throwable t) {
+ LOG.error("Failed to restartNodes OM {}", om, t);
+ }
+ });
+ }
+ }
+
+ /**
+ * Start/Stop Ozone Manager to induce failure.
+ */
+ public static class OzoneManagerStartStopFailure extends OzoneFailures {
+ public void fail(MiniOzoneChaosCluster cluster) {
+ // Get the number of OzoneManager to fail in the cluster.
+ boolean shouldStop = cluster.shouldStop();
+ Set<OzoneManager> oms = cluster.omToFail();
+ oms.parallelStream().forEach(om -> {
+ try {
+ if (shouldStop) {
+ // start another OM before failing the next one.
+ cluster.shutdownOzoneManager(om);
+ } else {
+ cluster.restartOzoneManager(om, true);
+ }
+ } catch (Throwable t) {
+ LOG.error("Failed to shutdown OM {}", om, t);
+ }
+ });
+ }
+ }
+
+ /**
+ * Datanode failures.
+ */
+ public abstract static class DatanodeFailures extends Failures {
+ @Override
+ public void validateFailure(MiniOzoneChaosCluster cluster) {
+ // Nothing to do here.
+ }
+ }
+
+ /**
+ * Restart Datanodes to induce failure.
+ */
+ public static class DatanodeRestartFailure extends DatanodeFailures {
+ public void fail(MiniOzoneChaosCluster cluster) {
+ boolean failureMode = FailureManager.isFastRestart();
+ Set<DatanodeDetails> dns = cluster.dnToFail();
+ dns.parallelStream().forEach(dn -> {
+ try {
+ cluster.restartHddsDatanode(dn, failureMode);
+ } catch (Throwable t) {
+ LOG.error("Failed to restartNodes Datanode {}", dn.getUuid(), t);
+ }
+ });
+ }
+ }
+
+ /**
+ * Start/Stop Datanodes to induce failure.
+ */
+ public static class DatanodeStartStopFailure extends DatanodeFailures {
+ public void fail(MiniOzoneChaosCluster cluster) {
+ // Get the number of datanodes to fail in the cluster.
+ Set<DatanodeDetails> dns = cluster.dnToFail();
+ dns.parallelStream().forEach(dn -> {
+ try {
+ if (cluster.shouldStop(dn)) {
+ cluster.shutdownHddsDatanode(dn);
+ } else {
+ cluster.restartHddsDatanode(dn, true);
+ }
+ } catch (Throwable t) {
+ LOG.error("Failed to shutdown Datanode {}", dn.getUuid(), t);
+ }
+ });
+ }
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/package-info.java b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/failure/package-info.java
similarity index 89%
copy from hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/package-info.java
copy to hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/failure/package-info.java
index 80c1985..e93958a 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/package-info.java
+++ b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/failure/package-info.java
@@ -6,16 +6,14 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- * <p>
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- * <p>
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.ozone.web.ozShell;
-/**
- * Tests for ozone shell..
- */
+
+package org.apache.hadoop.ozone.failure;
\ No newline at end of file
diff --git a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/AgedDirLoadGenerator.java b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/AgedDirLoadGenerator.java
new file mode 100644
index 0000000..f4ab930
--- /dev/null
+++ b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/AgedDirLoadGenerator.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.loadgenerators;
+
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.ozone.utils.LoadBucket;
+
+/**
+ * A load generator where directories are read multiple times.
+ */
+public class AgedDirLoadGenerator extends LoadGenerator {
+ private final LoadBucket fsBucket;
+ private final int maxDirIndex;
+
+ public AgedDirLoadGenerator(DataBuffer dataBuffer, LoadBucket fsBucket) {
+ this.fsBucket = fsBucket;
+ this.maxDirIndex = 100;
+ }
+
+ @Override
+ public void generateLoad() throws Exception {
+ int index = RandomUtils.nextInt(0, maxDirIndex);
+ String keyName = getKeyName(index);
+ fsBucket.readDirectory(keyName);
+ }
+
+ @Override
+ public void initialize() throws Exception {
+ for (int i = 0; i < maxDirIndex; i++) {
+ String keyName = getKeyName(i);
+ fsBucket.createDirectory(keyName);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/AgedLoadGenerator.java b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/AgedLoadGenerator.java
index 766343d..ecd6076 100644
--- a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/AgedLoadGenerator.java
+++ b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/AgedLoadGenerator.java
@@ -21,8 +21,6 @@
import org.apache.commons.lang3.RandomUtils;
import org.apache.hadoop.ozone.utils.LoadBucket;
import org.apache.hadoop.ozone.utils.TestProbability;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.nio.ByteBuffer;
import java.util.Optional;
@@ -37,10 +35,6 @@
*/
public class AgedLoadGenerator extends LoadGenerator {
- private static final Logger LOG =
- LoggerFactory.getLogger(AgedLoadGenerator.class);
- private static String agedSuffix = "aged";
-
private final AtomicInteger agedFileWrittenIndex;
private final AtomicInteger agedFileAllocationIndex;
private final LoadBucket agedLoadBucket;
@@ -56,25 +50,21 @@
}
@Override
- public String generateLoad() throws Exception {
+ public void generateLoad() throws Exception {
if (agedWriteProbability.isTrue()) {
synchronized (agedFileAllocationIndex) {
int index = agedFileAllocationIndex.getAndIncrement();
ByteBuffer buffer = dataBuffer.getBuffer(index);
- String keyName = getKeyName(index, agedSuffix);
+ String keyName = getKeyName(index);
agedLoadBucket.writeKey(buffer, keyName);
agedFileWrittenIndex.getAndIncrement();
- return keyName;
}
} else {
Optional<Integer> index = randomKeyToRead();
if (index.isPresent()) {
ByteBuffer buffer = dataBuffer.getBuffer(index.get());
- String keyName = getKeyName(index.get(), agedSuffix);
+ String keyName = getKeyName(index.get());
agedLoadBucket.readKey(buffer, keyName);
- return keyName;
- } else {
- return "NoKey";
}
}
}
@@ -90,9 +80,4 @@
public void initialize() {
// Nothing to do here
}
-
- @Override
- public String name() {
- return "Aged";
- }
}
diff --git a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/DataBuffer.java b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/DataBuffer.java
index 43126ee..14cc041 100644
--- a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/DataBuffer.java
+++ b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/DataBuffer.java
@@ -44,6 +44,7 @@
buffer.put(RandomUtils.nextBytes(size));
this.buffers.add(buffer);
}
+ // TODO: add buffers of sizes of prime numbers.
}
public ByteBuffer getBuffer(int keyIndex) {
diff --git a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/FilesystemLoadGenerator.java b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/FilesystemLoadGenerator.java
index 557c73b..e6cb7e5 100644
--- a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/FilesystemLoadGenerator.java
+++ b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/FilesystemLoadGenerator.java
@@ -20,8 +20,6 @@
import org.apache.commons.lang3.RandomUtils;
import org.apache.hadoop.ozone.utils.LoadBucket;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.nio.ByteBuffer;
@@ -32,9 +30,6 @@
* apis.
*/
public class FilesystemLoadGenerator extends LoadGenerator {
- private static final Logger LOG =
- LoggerFactory.getLogger(FilesystemLoadGenerator.class);
-
private final LoadBucket fsBucket;
private final DataBuffer dataBuffer;
@@ -45,25 +40,19 @@
}
@Override
- public String generateLoad() throws Exception {
+ public void generateLoad() throws Exception {
int index = RandomUtils.nextInt();
ByteBuffer buffer = dataBuffer.getBuffer(index);
- String keyName = getKeyName(index, name());
+ String keyName = getKeyName(index);
fsBucket.writeKey(true, buffer, keyName);
fsBucket.readKey(true, buffer, keyName);
fsBucket.deleteKey(true, keyName);
- return keyName;
}
@Override
public void initialize() {
// Nothing to do here
}
-
- @Override
- public String name() {
- return "FileSystem";
- }
}
diff --git a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/LoadExecutors.java b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/LoadExecutors.java
index 5e34fb4..be9507b 100644
--- a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/LoadExecutors.java
+++ b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/LoadExecutors.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.loadgenerators;
+import org.apache.commons.lang3.RandomUtils;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
@@ -38,31 +39,32 @@
private static final Logger LOG =
LoggerFactory.getLogger(LoadExecutors.class);
- private final LoadGenerator generator;
+ private final List<LoadGenerator> generators;
private final int numThreads;
private final ExecutorService executor;
+ private final int numGenerators;
private final List<CompletableFuture<Void>> futures = new ArrayList<>();
- public LoadExecutors(int numThreads, LoadGenerator generator) {
+ public LoadExecutors(int numThreads, List<LoadGenerator> generators) {
this.numThreads = numThreads;
- this.generator = generator;
+ this.generators = generators;
+ this.numGenerators = generators.size();
this.executor = Executors.newFixedThreadPool(numThreads);
}
private void load(long runTimeMillis) {
long threadID = Thread.currentThread().getId();
- LOG.info("{} LOADGEN: Started Aged IO Thread:{}.",
- generator.name(), threadID);
+ LOG.info("LOADGEN: Started IO Thread:{}.", threadID);
long startTime = Time.monotonicNow();
while (Time.monotonicNow() - startTime < runTimeMillis) {
+ LoadGenerator gen =
+ generators.get(RandomUtils.nextInt(0, numGenerators));
- String keyName = null;
try {
- keyName = generator.generateLoad();
+ gen.generateLoad();
} catch (Throwable t) {
- LOG.error("{} LOADGEN: {} Exiting due to exception",
- generator.name(), keyName, t);
+ LOG.error("{} LOADGEN: Exiting due to exception", gen, t);
ExitUtil.terminate(new ExitUtil.ExitException(1, t));
break;
}
@@ -70,12 +72,21 @@
}
- public void startLoad(long time) {
- LOG.info("Starting {} threads for {}", numThreads, generator.name());
- generator.initialize();
+ public void startLoad(long time) throws Exception {
+ LOG.info("Starting {} threads for {} genrators", numThreads,
+ generators.size());
+ for (LoadGenerator gen : generators) {
+ try {
+ LOG.info("Initializing {} generator", gen);
+ gen.initialize();
+ } catch (Throwable t) {
+ LOG.error("Failed to initialize loadgen:{}", gen, t);
+ throw t;
+ }
+ }
+
for (int i = 0; i < numThreads; i++) {
- futures.add(CompletableFuture.runAsync(
- () -> load(time), executor));
+ futures.add(CompletableFuture.runAsync(() -> load(time), executor));
}
}
diff --git a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/LoadGenerator.java b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/LoadGenerator.java
index 014a46f..7f79df5 100644
--- a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/LoadGenerator.java
+++ b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/LoadGenerator.java
@@ -22,16 +22,29 @@
* Interface for load generator.
*/
public abstract class LoadGenerator {
+ /*
+ * The implemented LoadGenerators constructors should have the
+ * constructor with the signature as following
+ * class NewLoadGen implements LoadGenerator {
+ *
+ * NewLoadGen(DataBuffer buffer, LoadBucket bucket) {
+ * // Add code here
+ * }
+ * }
+ */
private final String keyNameDelimiter = "_";
- public abstract void initialize();
+ public abstract void initialize() throws Exception;
- public abstract String generateLoad() throws Exception;
+ public abstract void generateLoad() throws Exception;
- public abstract String name();
+ String getKeyName(int keyIndex) {
+ return toString() + keyNameDelimiter + keyIndex;
+ }
- String getKeyName(int keyIndex, String prefix) {
- return prefix + keyNameDelimiter + keyIndex;
+ @Override
+ public String toString() {
+ return this.getClass().getSimpleName();
}
}
diff --git a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/NestedDirLoadGenerator.java b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/NestedDirLoadGenerator.java
new file mode 100644
index 0000000..ded85a7
--- /dev/null
+++ b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/NestedDirLoadGenerator.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.loadgenerators;
+
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.ozone.utils.LoadBucket;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * A Load generator where nested directories are created and read them.
+ */
+public class NestedDirLoadGenerator extends LoadGenerator {
+ private final LoadBucket fsBucket;
+ private final int maxDirDepth;
+ private final Map<Integer, String> pathMap;
+
+ public NestedDirLoadGenerator(DataBuffer dataBuffer, LoadBucket fsBucket) {
+ this.fsBucket = fsBucket;
+ this.maxDirDepth = 20;
+ this.pathMap = new ConcurrentHashMap<>();
+ }
+
+ private String createNewPath(int i, String s) {
+ String base = s != null ? s : "";
+ return base + "/" + getKeyName(i);
+ }
+
+ @Override
+ public void generateLoad() throws Exception {
+ int index = RandomUtils.nextInt(0, maxDirDepth);
+ String str = this.pathMap.compute(index, this::createNewPath);
+ fsBucket.createDirectory(str);
+ fsBucket.readDirectory(str);
+ }
+
+ @Override
+ public void initialize() throws Exception {
+ // Nothing to do here
+ }
+}
\ No newline at end of file
diff --git a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/RandomDirLoadGenerator.java b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/RandomDirLoadGenerator.java
new file mode 100644
index 0000000..8eaba65
--- /dev/null
+++ b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/RandomDirLoadGenerator.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.loadgenerators;
+
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.ozone.utils.LoadBucket;
+
+/**
+ * A simple directory based load generator.
+ */
+public class RandomDirLoadGenerator extends LoadGenerator {
+ private final LoadBucket fsBucket;
+
+ public RandomDirLoadGenerator(DataBuffer dataBuffer, LoadBucket fsBucket) {
+ this.fsBucket = fsBucket;
+ }
+
+ @Override
+ public void generateLoad() throws Exception {
+ int index = RandomUtils.nextInt();
+ String keyName = getKeyName(index);
+ fsBucket.createDirectory(keyName);
+ fsBucket.readDirectory(keyName);
+ }
+
+ @Override
+ public void initialize() {
+ // Nothing to do here
+ }
+}
\ No newline at end of file
diff --git a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/RandomLoadGenerator.java b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/RandomLoadGenerator.java
index a9fc41c..7d856ac 100644
--- a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/RandomLoadGenerator.java
+++ b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/RandomLoadGenerator.java
@@ -24,7 +24,6 @@
import org.slf4j.LoggerFactory;
import java.nio.ByteBuffer;
-import java.util.List;
/**
* Random load generator which writes, read and deletes keys from
@@ -34,35 +33,27 @@
private static final Logger LOG =
LoggerFactory.getLogger(RandomLoadGenerator.class);
- private final List<LoadBucket> ozoneBuckets;
+ private final LoadBucket ozoneBucket;
private final DataBuffer dataBuffer;
- public RandomLoadGenerator(DataBuffer dataBuffer, List<LoadBucket> buckets) {
- this.ozoneBuckets = buckets;
+ public RandomLoadGenerator(DataBuffer dataBuffer, LoadBucket bucket) {
+ this.ozoneBucket = bucket;
this.dataBuffer = dataBuffer;
}
@Override
- public String generateLoad() throws Exception {
- LoadBucket bucket =
- ozoneBuckets.get((int) (Math.random() * ozoneBuckets.size()));
+ public void generateLoad() throws Exception {
int index = RandomUtils.nextInt();
ByteBuffer buffer = dataBuffer.getBuffer(index);
- String keyName = getKeyName(index, name());
- bucket.writeKey(buffer, keyName);
+ String keyName = getKeyName(index);
+ ozoneBucket.writeKey(buffer, keyName);
- bucket.readKey(buffer, keyName);
+ ozoneBucket.readKey(buffer, keyName);
- bucket.deleteKey(keyName);
- return keyName;
+ ozoneBucket.deleteKey(keyName);
}
public void initialize() {
// Nothing to do here
}
-
- @Override
- public String name() {
- return "Random";
- }
}
diff --git a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/ReadOnlyLoadGenerator.java b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/ReadOnlyLoadGenerator.java
new file mode 100644
index 0000000..8397800
--- /dev/null
+++ b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/ReadOnlyLoadGenerator.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.loadgenerators;
+
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.ozone.utils.LoadBucket;
+
+import java.nio.ByteBuffer;
+
+/**
+ * This load generator writes some files and reads the same file multiple times.
+ */
+public class ReadOnlyLoadGenerator extends LoadGenerator {
+ private final LoadBucket replBucket;
+ private final DataBuffer dataBuffer;
+ private final static int NUM_KEYS = 10;
+
+ public ReadOnlyLoadGenerator(DataBuffer dataBuffer, LoadBucket replBucket) {
+ this.dataBuffer = dataBuffer;
+ this.replBucket = replBucket;
+ }
+
+ @Override
+ public void generateLoad() throws Exception {
+ int index = RandomUtils.nextInt(0, NUM_KEYS);
+ ByteBuffer buffer = dataBuffer.getBuffer(index);
+ String keyName = getKeyName(index);
+ replBucket.readKey(buffer, keyName);
+ }
+
+
+ public void initialize() throws Exception {
+ for (int index = 0; index < NUM_KEYS; index++) {
+ ByteBuffer buffer = dataBuffer.getBuffer(index);
+ String keyName = getKeyName(index);
+ replBucket.writeKey(buffer, keyName);
+ }
+ }
+}
diff --git a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/utils/LoadBucket.java b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/utils/LoadBucket.java
index 8e1ef31..51c344f 100644
--- a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/utils/LoadBucket.java
+++ b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/utils/LoadBucket.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.ozone.utils;
import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.ozone.OzoneFileSystem;
@@ -29,6 +30,7 @@
import org.apache.hadoop.ozone.client.OzoneBucket;
import java.io.InputStream;
import java.io.OutputStream;
+import org.junit.Assert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -76,6 +78,16 @@
writeOp.execute();
}
+ public void createDirectory(String keyName) throws Exception {
+ Op dirOp = new DirectoryOp(keyName, false);
+ dirOp.execute();
+ }
+
+ public void readDirectory(String keyName) throws Exception {
+ Op dirOp = new DirectoryOp(keyName, true);
+ dirOp.execute();
+ }
+
// Read ops.
public void readKey(ByteBuffer buffer, String keyName) throws Exception {
readKey(isFsOp(), buffer, keyName);
@@ -148,6 +160,46 @@
}
/**
+ * Create and Read Directories.
+ */
+ public class DirectoryOp extends Op {
+ private final boolean readDir;
+
+ DirectoryOp(String keyName, boolean readDir) {
+ super(true, keyName);
+ this.readDir = readDir;
+ }
+
+ @Override
+ void doFsOp(Path p) throws IOException {
+ if (readDir) {
+ FileStatus status = fs.getFileStatus(p);
+ Assert.assertTrue(status.isDirectory());
+ Assert.assertEquals(p,
+ Path.getPathWithoutSchemeAndAuthority(status.getPath()));
+ } else {
+ Assert.assertTrue(fs.mkdirs(p));
+ }
+ }
+
+ @Override
+ void doBucketOp(String key) throws IOException {
+ // nothing to do here
+ }
+
+ @Override
+ void doPostOp() throws IOException {
+ // Nothing to do here
+ }
+
+ @Override
+ public String toString() {
+ return super.toString() + " "
+ + (readDir ? "readDirectory": "writeDirectory");
+ }
+ }
+
+ /**
* Write file/key to bucket.
*/
public class WriteOp extends Op {
diff --git a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/resources/log4j.properties b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/resources/log4j.properties
index f491fad..aabb0b1 100644
--- a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/resources/log4j.properties
+++ b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/resources/log4j.properties
@@ -24,7 +24,8 @@
log4j.logger.org.apache.ratis.grpc.client.GrpcClientProtocolClient=WARN
log4j.logger.org.apache.hadoop.ozone.utils=DEBUG,stdout,CHAOS
-log4j.logger.org.apache.hadoop.ozone.loadgenerator=DEBUG,stdout,CHAOS
+log4j.logger.org.apache.hadoop.ozone.loadgenerators=DEBUG,stdout,CHAOS
+log4j.logger.org.apache.hadoop.ozone.failure=INFO, CHAOS
log4j.appender.CHAOS.File=${chaoslogfilename}
log4j.appender.CHAOS=org.apache.log4j.FileAppender
log4j.appender.CHAOS.layout=org.apache.log4j.PatternLayout
diff --git a/hadoop-ozone/fault-injection-test/network-tests/pom.xml b/hadoop-ozone/fault-injection-test/network-tests/pom.xml
index b361e55..49c5098 100644
--- a/hadoop-ozone/fault-injection-test/network-tests/pom.xml
+++ b/hadoop-ozone/fault-injection-test/network-tests/pom.xml
@@ -31,7 +31,6 @@
<plugins>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
- <version>3.1.0</version>
<executions>
<execution>
<id>copy-resources</id>
diff --git a/hadoop-ozone/insight/pom.xml b/hadoop-ozone/insight/pom.xml
index c66304e..f401750 100644
--- a/hadoop-ozone/insight/pom.xml
+++ b/hadoop-ozone/insight/pom.xml
@@ -37,6 +37,11 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-common</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-test-utils</artifactId>
+ <scope>test</scope>
+ </dependency>
<!-- Genesis requires server side components -->
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -56,13 +61,7 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <scope>compile</scope>
+ <artifactId>hadoop-hdds-tools</artifactId>
</dependency>
<dependency>
<groupId>com.sun.xml.bind</groupId>
@@ -79,17 +78,6 @@
<dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
- <version>3.2.4</version>
- </dependency>
- <dependency>
- <groupId>org.openjdk.jmh</groupId>
- <artifactId>jmh-core</artifactId>
- <version>1.19</version>
- </dependency>
- <dependency>
- <groupId>org.openjdk.jmh</groupId>
- <artifactId>jmh-generator-annprocess</artifactId>
- <version>1.19</version>
</dependency>
<dependency>
<groupId>com.github.spotbugs</groupId>
@@ -103,9 +91,8 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
+ <artifactId>hadoop-hdds-hadoop-dependency-test</artifactId>
<scope>test</scope>
- <type>test-jar</type>
</dependency>
</dependencies>
<build>
diff --git a/hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/LogSubcommand.java b/hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/LogSubcommand.java
index e306f42..5e487a5 100644
--- a/hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/LogSubcommand.java
+++ b/hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/LogSubcommand.java
@@ -110,6 +110,8 @@
try {
thread.join();
} catch (InterruptedException e) {
+ // restore thread interrupted state
+ Thread.currentThread().interrupt();
e.printStackTrace();
}
}
diff --git a/hadoop-ozone/integration-test/pom.xml b/hadoop-ozone/integration-test/pom.xml
index b8d5800..622a407 100644
--- a/hadoop-ozone/integration-test/pom.xml
+++ b/hadoop-ozone/integration-test/pom.xml
@@ -36,6 +36,11 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-test-utils</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-server-scm</artifactId>
</dependency>
<dependency>
@@ -73,8 +78,12 @@
<artifactId>hadoop-ozone-tools</artifactId>
</dependency>
<dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-tools</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
<scope>test</scope>
</dependency>
<dependency>
@@ -101,6 +110,7 @@
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
+ <version>${mockito1-hadoop.version}</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -128,9 +138,8 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
+ <artifactId>hadoop-hdds-hadoop-dependency-test</artifactId>
<scope>test</scope>
- <type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
index 20e6104..1233e0b 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
@@ -62,6 +62,7 @@
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -404,9 +405,10 @@
// For directories, the time returned is the current time when the dir key
// doesn't actually exist on server; if it exists, it will be a fixed value.
// In this case, the dir key exists.
- assertEquals(0, omStatus.getLen());
- assertTrue(omStatus.getModificationTime() <= currentTime);
- assertEquals(omStatus.getPath().getName(), o3fs.pathToKey(path));
+ assertEquals(0, omStatus.getKeyInfo().getDataSize());
+ assertTrue(omStatus.getKeyInfo().getModificationTime() <= currentTime);
+ assertEquals(new Path(omStatus.getPath()).getName(),
+ o3fs.pathToKey(path));
}
@Test
@@ -429,6 +431,7 @@
}
@Test
+ @Ignore("HDDS-3506")
public void testOzoneManagerLocatedFileStatusBlockOffsetsWithMultiBlockFile()
throws Exception {
// naive assumption: MiniOzoneCluster will not have larger than ~1GB
@@ -438,7 +441,7 @@
OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT,
StorageUnit.BYTES
);
- String data = RandomStringUtils.randomAlphanumeric(2*blockSize+837);
+ String data = RandomStringUtils.randomAlphanumeric(2 * blockSize + 837);
String filePath = RandomStringUtils.randomAlphanumeric(5);
Path path = createPath("/" + filePath);
try (FSDataOutputStream stream = fs.create(path)) {
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
index bc0e752..e9324e6 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
@@ -21,9 +21,12 @@
import java.io.IOException;
import java.util.Set;
import java.util.TreeSet;
+import java.util.concurrent.TimeoutException;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -42,6 +45,7 @@
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import org.apache.hadoop.test.LambdaTestUtils;
import org.junit.After;
@@ -65,24 +69,81 @@
private int rootItemCount;
@Test(timeout = 300_000)
+ public void testCreateFileShouldCheckExistenceOfDirWithSameName()
+ throws Exception {
+ /*
+ * Op 1. create file -> /d1/d2/d3/d4/key2
+ * Op 2. create dir -> /d1/d2/d3/d4/key2
+ *
+ * Reverse of the above steps
+ * Op 2. create dir -> /d1/d2/d3/d4/key3
+ * Op 1. create file -> /d1/d2/d3/d4/key3
+ *
+ * Op 3. create file -> /d1/d2/d3 (d3 as a file inside /d1/d2)
+ */
+ setupOzoneFileSystem();
+
+ Path parent = new Path("/d1/d2/d3/d4/");
+ Path file1 = new Path(parent, "key1");
+ try (FSDataOutputStream outputStream = fs.create(file1, false)) {
+ assertNotNull("Should be able to create file", outputStream);
+ }
+
+ Path dir1 = new Path("/d1/d2/d3/d4/key2");
+ fs.mkdirs(dir1);
+ try (FSDataOutputStream outputStream1 = fs.create(dir1, false)) {
+ fail("Should throw FileAlreadyExistsException");
+ } catch (FileAlreadyExistsException fae){
+ // ignore as its expected
+ }
+
+ Path file2 = new Path("/d1/d2/d3/d4/key3");
+ try (FSDataOutputStream outputStream2 = fs.create(file2, false)) {
+ assertNotNull("Should be able to create file", outputStream2);
+ }
+ try {
+ fs.mkdirs(file2);
+ fail("Should throw FileAlreadyExistsException");
+ } catch (FileAlreadyExistsException fae) {
+ // ignore as its expected
+ }
+
+ // Op 3. create file -> /d1/d2/d3 (d3 as a file inside /d1/d2)
+ Path file3 = new Path("/d1/d2/d3");
+ try (FSDataOutputStream outputStream2 = fs.create(file3, false)) {
+ fail("Should throw FileAlreadyExistsException");
+ } catch (FileAlreadyExistsException fae) {
+ // ignore as its expected
+ }
+ }
+
+ /**
+ * Make the given file and all non-existent parents into
+ * directories. Has roughly the semantics of Unix @{code mkdir -p}.
+ * {@link FileSystem#mkdirs(Path)}
+ */
+ @Test(timeout = 300_000)
+ public void testMakeDirsWithAnExistingDirectoryPath() throws Exception {
+ /*
+ * Op 1. create file -> /d1/d2/d3/d4/k1 (d3 is a sub-dir inside /d1/d2)
+ * Op 2. create dir -> /d1/d2
+ */
+ setupOzoneFileSystem();
+
+ Path parent = new Path("/d1/d2/d3/d4/");
+ Path file1 = new Path(parent, "key1");
+ try (FSDataOutputStream outputStream = fs.create(file1, false)) {
+ assertNotNull("Should be able to create file", outputStream);
+ }
+
+ Path subdir = new Path("/d1/d2/");
+ boolean status = fs.mkdirs(subdir);
+ assertTrue("Shouldn't send error if dir exists", status);
+ }
+
+ @Test(timeout = 300_000)
public void testFileSystem() throws Exception {
- OzoneConfiguration conf = new OzoneConfiguration();
- cluster = MiniOzoneCluster.newBuilder(conf)
- .setNumDatanodes(3)
- .build();
- cluster.waitForClusterToBeReady();
-
- // create a volume and a bucket to be used by OzoneFileSystem
- OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
- volumeName = bucket.getVolumeName();
- bucketName = bucket.getName();
-
- String rootPath = String.format("%s://%s.%s/",
- OzoneConsts.OZONE_URI_SCHEME, bucket.getName(), bucket.getVolumeName());
-
- // Set the fs.defaultFS and start the filesystem
- conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
- fs = FileSystem.get(conf);
+ setupOzoneFileSystem();
testOzoneFsServiceLoader();
o3fs = (OzoneFileSystem) fs;
@@ -111,6 +172,27 @@
}
}
+ private void setupOzoneFileSystem()
+ throws IOException, TimeoutException, InterruptedException {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(3)
+ .build();
+ cluster.waitForClusterToBeReady();
+ // create a volume and a bucket to be used by OzoneFileSystem
+ OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
+ volumeName = bucket.getVolumeName();
+ bucketName = bucket.getName();
+
+ String rootPath = String.format("%s://%s.%s/",
+ OzoneConsts.OZONE_URI_SCHEME, bucket.getName(),
+ bucket.getVolumeName());
+
+ // Set the fs.defaultFS and start the filesystem
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
+ fs = FileSystem.get(conf);
+ }
+
private void testOzoneFsServiceLoader() throws IOException {
assertEquals(
FileSystem.getFileSystemClass(OzoneConsts.OZONE_URI_SCHEME, null),
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java
index e57d3bb..cfe0b3a 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java
@@ -49,7 +49,6 @@
import java.util.Optional;
import java.util.OptionalInt;
import java.util.UUID;
-import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.hdds.HddsUtils.getHostName;
import static org.apache.hadoop.hdds.HddsUtils.getHostPort;
@@ -79,8 +78,6 @@
private final String o3fsImplValue =
"org.apache.hadoop.fs.ozone.OzoneFileSystem";
- private static final long LEADER_ELECTION_TIMEOUT = 500L;
-
@Before
public void init() throws Exception {
conf = new OzoneConfiguration();
@@ -93,10 +90,6 @@
java.nio.file.Path metaDirPath = java.nio.file.Paths.get(path, "om-meta");
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDirPath.toString());
conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "127.0.0.1:0");
- conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, true);
- conf.setTimeDuration(
- OMConfigKeys.OZONE_OM_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY,
- LEADER_ELECTION_TIMEOUT, TimeUnit.MILLISECONDS);
conf.setInt(ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT, 3);
OMStorage omStore = new OMStorage(conf);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java
index dd54315..19ff428 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java
@@ -18,14 +18,15 @@
package org.apache.hadoop.fs.ozone.contract;
+import java.io.IOException;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
import org.apache.hadoop.fs.contract.AbstractFSContract;
+
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import java.io.IOException;
-
/**
* Ozone contract tests creating files.
*/
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java
index f0a3d8d..33e6260 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java
@@ -18,14 +18,15 @@
package org.apache.hadoop.fs.ozone.contract;
+import java.io.IOException;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
import org.apache.hadoop.fs.contract.AbstractFSContract;
+
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import java.io.IOException;
-
/**
* Ozone contract tests covering deletes.
*/
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java
index 134a9ad..ce63456 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java
@@ -18,13 +18,14 @@
package org.apache.hadoop.fs.ozone.contract;
+import java.io.IOException;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.tools.contract.AbstractContractDistCpTest;
+
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import java.io.IOException;
-
/**
* Contract test suite covering S3A integration with DistCp.
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java
index 362b22f..9d9aa56 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java
@@ -18,16 +18,17 @@
package org.apache.hadoop.fs.ozone.contract;
+import java.io.IOException;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest;
import org.apache.hadoop.fs.contract.AbstractFSContract;
+
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-
/**
* Ozone contract tests covering getFileStatus.
*/
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java
index bc0de5d..305164c 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java
@@ -18,14 +18,15 @@
package org.apache.hadoop.fs.ozone.contract;
+import java.io.IOException;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
import org.apache.hadoop.fs.contract.AbstractFSContract;
+
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import java.io.IOException;
-
/**
* Test dir operations on Ozone.
*/
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java
index 0bc57d4..aa81965 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java
@@ -18,14 +18,15 @@
package org.apache.hadoop.fs.ozone.contract;
+import java.io.IOException;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
import org.apache.hadoop.fs.contract.AbstractFSContract;
+
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import java.io.IOException;
-
/**
* Ozone contract tests opening files.
*/
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java
index 8ce1d1b..3660d81 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java
@@ -18,14 +18,15 @@
package org.apache.hadoop.fs.ozone.contract;
+import java.io.IOException;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
import org.apache.hadoop.fs.contract.AbstractFSContract;
+
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import java.io.IOException;
-
/**
* Ozone contract tests covering rename.
*/
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java
index 3156eb2..c64dafa 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java
@@ -18,14 +18,14 @@
package org.apache.hadoop.fs.ozone.contract;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
+import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
import org.apache.hadoop.fs.contract.AbstractFSContract;
-import java.io.IOException;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
/**
* Ozone contract test for ROOT directory operations.
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java
index c4bc0ff..2f22025 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java
@@ -18,14 +18,15 @@
package org.apache.hadoop.fs.ozone.contract;
+import java.io.IOException;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
import org.apache.hadoop.fs.contract.AbstractFSContract;
+
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import java.io.IOException;
-
/**
* Ozone contract tests covering file seek.
*/
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java
index 35d1774..33b2268 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java
@@ -18,42 +18,32 @@
package org.apache.hadoop.hdds.scm.pipeline;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.ratis.RatisHelper;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.container.ContainerManager;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
import java.io.IOException;
+import java.util.List;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import static org.apache.hadoop.hdds.protocol.proto.HddsProtos
- .ReplicationFactor.THREE;
-import static org.apache.hadoop.hdds.protocol.proto.HddsProtos
- .ReplicationType.RATIS;
/**
* Test Node failure detection and handling in Ratis.
*/
-@Ignore
public class TestNodeFailure {
private static MiniOzoneCluster cluster;
- private static OzoneConfiguration conf;
- private static Pipeline ratisPipelineOne;
- private static Pipeline ratisPipelineTwo;
- private static ContainerManager containerManager;
+ private static List<Pipeline> ratisPipelines;
private static PipelineManager pipelineManager;
- private static long timeForFailure;
+ private static int timeForFailure;
/**
* Create a MiniDFSCluster for testing.
@@ -62,7 +52,7 @@
*/
@BeforeClass
public static void init() throws Exception {
- conf = new OzoneConfiguration();
+ final OzoneConfiguration conf = new OzoneConfiguration();
conf.setTimeDuration(
RatisHelper.HDDS_DATANODE_RATIS_SERVER_PREFIX_KEY + "." +
DatanodeRatisServerConfig.RATIS_FOLLOWER_SLOWNESS_TIMEOUT_KEY,
@@ -71,28 +61,22 @@
RatisHelper.HDDS_DATANODE_RATIS_SERVER_PREFIX_KEY + "." +
DatanodeRatisServerConfig.RATIS_SERVER_NO_LEADER_TIMEOUT_KEY,
10, TimeUnit.SECONDS);
- conf.setTimeDuration(
- ScmConfigKeys.OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT,
- 10, TimeUnit.SECONDS);
+ conf.set(HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL, "2s");
+
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(6)
.setHbInterval(1000)
.setHbProcessorInterval(1000)
.build();
cluster.waitForClusterToBeReady();
- StorageContainerManager scm = cluster.getStorageContainerManager();
- containerManager = scm.getContainerManager();
+
+ final StorageContainerManager scm = cluster.getStorageContainerManager();
pipelineManager = scm.getPipelineManager();
- ratisPipelineOne = pipelineManager.getPipeline(
- containerManager.allocateContainer(
- RATIS, THREE, "testOwner").getPipelineID());
- ratisPipelineTwo = pipelineManager.getPipeline(
- containerManager.allocateContainer(
- RATIS, THREE, "testOwner").getPipelineID());
- // At this stage, there should be 2 pipeline one with 1 open container each.
- // Try closing the both the pipelines, one with a closed container and
- // the other with an open container.
- timeForFailure = conf.getObject(DatanodeRatisServerConfig.class)
+ ratisPipelines = pipelineManager.getPipelines(
+ HddsProtos.ReplicationType.RATIS,
+ HddsProtos.ReplicationFactor.THREE);
+
+ timeForFailure = (int) conf.getObject(DatanodeRatisServerConfig.class)
.getFollowerSlownessTimeout();
}
@@ -106,35 +90,39 @@
}
}
- @Ignore
- // Enable this after we implement teardown pipeline logic once a datanode
- // dies.
- @Test(timeout = 300_000L)
- public void testPipelineFail() throws InterruptedException, IOException,
- TimeoutException {
- Assert.assertEquals(ratisPipelineOne.getPipelineState(),
- Pipeline.PipelineState.OPEN);
- Pipeline pipelineToFail = ratisPipelineOne;
- DatanodeDetails dnToFail = pipelineToFail.getFirstNode();
- cluster.shutdownHddsDatanode(dnToFail);
+ @Test
+ public void testPipelineFail() {
+ ratisPipelines.forEach(pipeline -> {
+ try {
+ waitForPipelineCreation(pipeline.getId());
+ cluster.shutdownHddsDatanode(pipeline.getFirstNode());
+ GenericTestUtils.waitFor(() -> {
+ try {
+ return pipelineManager.getPipeline(pipeline.getId())
+ .getPipelineState().equals(Pipeline.PipelineState.CLOSED);
+ } catch (PipelineNotFoundException ex) {
+ return true;
+ }
+ }, timeForFailure / 2, timeForFailure * 3);
+ } catch (Exception e) {
+ Assert.fail("Test Failed: " + e.getMessage());
+ }
+ });
+ }
- // wait for sufficient time for the callback to be triggered
- Thread.sleep(3 * timeForFailure);
-
- Assert.assertEquals(Pipeline.PipelineState.CLOSED,
- pipelineManager.getPipeline(ratisPipelineOne.getId())
- .getPipelineState());
- Assert.assertEquals(Pipeline.PipelineState.OPEN,
- pipelineManager.getPipeline(ratisPipelineTwo.getId())
- .getPipelineState());
- // Now restart the datanode and make sure that a new pipeline is created.
- cluster.setWaitForClusterToBeReadyTimeout(300000);
- cluster.restartHddsDatanode(dnToFail, true);
- Pipeline ratisPipelineThree = pipelineManager.getPipeline(
- containerManager.allocateContainer(
- RATIS, THREE, "testOwner").getPipelineID());
- //Assert that new container is not created from the ratis 2 pipeline
- Assert.assertNotEquals(ratisPipelineThree.getId(),
- ratisPipelineTwo.getId());
+ /**
+ * Waits until the Pipeline is marked as OPEN.
+ * @param pipelineID Id of the pipeline
+ */
+ private void waitForPipelineCreation(final PipelineID pipelineID)
+ throws Exception {
+ GenericTestUtils.waitFor(() -> {
+ try {
+ return pipelineManager.getPipeline(pipelineID)
+ .getPipelineState().equals(Pipeline.PipelineState.OPEN);
+ } catch (PipelineNotFoundException ex) {
+ return false;
+ }
+ }, 1000, 1000 * 60);
}
}
\ No newline at end of file
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index 1d1ea90..d1d5162 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -17,21 +17,6 @@
*/
package org.apache.hadoop.ozone;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.StorageUnit;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
-import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.om.OzoneManager;
-import org.apache.hadoop.hdds.scm.protocolPB
- .StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.ozone.recon.ReconServer;
-import org.apache.hadoop.security.authentication.client.AuthenticationException;
-import org.apache.hadoop.test.GenericTestUtils;
-
import java.io.IOException;
import java.util.List;
import java.util.Optional;
@@ -39,6 +24,19 @@
import java.util.UUID;
import java.util.concurrent.TimeoutException;
+import org.apache.hadoop.conf.StorageUnit;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.recon.ReconServer;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.test.GenericTestUtils;
+
/**
* Interface used for MiniOzoneClusters.
*/
@@ -71,7 +69,7 @@
*
* @return Configuration
*/
- Configuration getConf();
+ OzoneConfiguration getConf();
/**
* Waits for the cluster to be ready, this call blocks till all the
@@ -305,6 +303,7 @@
protected int numOfOmHandlers = 20;
protected int numOfScmHandlers = 20;
protected int numOfDatanodes = 3;
+ protected int numDataVolumes = 1;
protected boolean startDataNodes = true;
protected CertificateClient certClient;
protected int pipelineNumLimit = DEFAULT_PIPELIME_LIMIT;
@@ -395,6 +394,18 @@
}
/**
+ * Sets the number of data volumes per datanode.
+ *
+ * @param val number of volumes per datanode.
+ *
+ * @return MiniOzoneCluster.Builder
+ */
+ public Builder setNumDataVolumes(int val) {
+ numDataVolumes = val;
+ return this;
+ }
+
+ /**
* Sets the total number of pipelines to create.
* @param val number of pipelines
* @return MiniOzoneCluster.Builder
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 98a6eff..5baa65b 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -29,7 +29,7 @@
import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_HTTP_ADDRESS_KEY;
import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_OM_SNAPSHOT_DB_DIR;
import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SCM_DB_DIR;
-import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SQL_DB_JDBC_URL;
+import static org.hadoop.ozone.recon.codegen.ReconSqlDbConfig.ConfigKeys.OZONE_RECON_SQL_DB_JDBC_URL;
import java.io.File;
import java.io.IOException;
@@ -699,21 +699,25 @@
OzoneConfiguration dnConf = new OzoneConfiguration(conf);
String datanodeBaseDir = path + "/datanode-" + Integer.toString(i);
Path metaDir = Paths.get(datanodeBaseDir, "meta");
- Path dataDir = Paths.get(datanodeBaseDir, "data", "containers");
+ List<String> dataDirs = new ArrayList<>();
+ for (int j = 0; j < numDataVolumes; j++) {
+ Path dir = Paths.get(datanodeBaseDir, "data-" + j, "containers");
+ Files.createDirectories(dir);
+ dataDirs.add(dir.toString());
+ }
+ String listOfDirs = String.join(",", dataDirs);
Path ratisDir = Paths.get(datanodeBaseDir, "data", "ratis");
- Path wrokDir = Paths.get(datanodeBaseDir, "data", "replication",
+ Path workDir = Paths.get(datanodeBaseDir, "data", "replication",
"work");
Files.createDirectories(metaDir);
- Files.createDirectories(dataDir);
Files.createDirectories(ratisDir);
- Files.createDirectories(wrokDir);
+ Files.createDirectories(workDir);
dnConf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.toString());
- dnConf.set(DFSConfigKeysLegacy.DFS_DATANODE_DATA_DIR_KEY,
- dataDir.toString());
+ dnConf.set(DFSConfigKeysLegacy.DFS_DATANODE_DATA_DIR_KEY, listOfDirs);
dnConf.set(OzoneConfigKeys.DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR,
ratisDir.toString());
dnConf.set(OzoneConfigKeys.OZONE_CONTAINER_COPY_WORKDIR,
- wrokDir.toString());
+ workDir.toString());
if (reconServer != null) {
OzoneStorageContainerManager reconScm =
reconServer.getReconStorageContainerManager();
@@ -800,8 +804,8 @@
.getAbsolutePath());
conf.set(OZONE_RECON_SCM_DB_DIR,
tempNewFolder.getAbsolutePath());
- conf.set(OZONE_RECON_SQL_DB_JDBC_URL, "jdbc:sqlite:" +
- tempNewFolder.getAbsolutePath() + "/ozone_recon_sqlite.db");
+ conf.set(OZONE_RECON_SQL_DB_JDBC_URL, "jdbc:derby:" +
+ tempNewFolder.getAbsolutePath() + "/ozone_recon_derby.db");
conf.set(OZONE_RECON_HTTP_ADDRESS_KEY, "0.0.0.0:0");
conf.set(OZONE_RECON_DATANODE_ADDRESS_KEY, "0.0.0.0:0");
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
index 56c9be1..6953594 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
@@ -69,8 +69,7 @@
private int waitForOMToBeReadyTimeout = 120000; // 2 min
private static final Random RANDOM = new Random();
- private static final int RATIS_LEADER_ELECTION_TIMEOUT = 1000; // 1 seconds
-
+ private static final int RATIS_LEADER_ELECTION_TIMEOUT = 1000; // 1 second
public static final int NODE_FAILURE_TIMEOUT = 2000; // 2 seconds
/**
@@ -207,20 +206,13 @@
}
}
- void shutdownOzoneManager(int omNodeIndex) {
- OzoneManager ozoneManager = ozoneManagers.get(omNodeIndex);
+ public void shutdownOzoneManager(OzoneManager ozoneManager) {
LOG.info("Shutting down OzoneManager " + ozoneManager.getOMNodeId());
ozoneManager.stop();
}
- void restartOzoneManager(int omNodeIndex, boolean waitForOM)
- throws IOException, TimeoutException, InterruptedException {
- OzoneManager ozoneManager = ozoneManagers.get(omNodeIndex);
- restartOzoneManager(ozoneManager, waitForOM);
- }
-
- void restartOzoneManager(OzoneManager ozoneManager, boolean waitForOM)
+ public void restartOzoneManager(OzoneManager ozoneManager, boolean waitForOM)
throws IOException, TimeoutException, InterruptedException {
LOG.info("Restarting OzoneManager " + ozoneManager.getOMNodeId());
ozoneManager.restart();
@@ -245,10 +237,12 @@
public void stopOzoneManager(int index) {
ozoneManagers.get(index).stop();
+ ozoneManagers.get(index).join();
}
public void stopOzoneManager(String omNodeId) {
ozoneManagerMap.get(omNodeId).stop();
+ ozoneManagerMap.get(omNodeId).join();
}
/**
@@ -314,16 +308,32 @@
protected void initOMRatisConf() {
conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, true);
conf.setInt(OMConfigKeys.OZONE_OM_HANDLER_COUNT_KEY, numOfOmHandlers);
- conf.setLong(
- OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_AUTO_TRIGGER_THRESHOLD_KEY,
- 100L);
- conf.setLong(OMConfigKeys.OZONE_OM_RATIS_LOG_PURGE_GAP, 200L);
+ // If test change the following config values we will respect,
+ // otherwise we will set lower timeout values.
+ long defaultDuration =
+ OMConfigKeys.OZONE_OM_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_DEFAULT
+ .getDuration();
+ long curLeaderElectionTimeout = conf.getTimeDuration(
+ OMConfigKeys.OZONE_OM_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY,
+ defaultDuration, TimeUnit.MILLISECONDS);
conf.setTimeDuration(
OMConfigKeys.OZONE_OM_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY,
- RATIS_LEADER_ELECTION_TIMEOUT, TimeUnit.MILLISECONDS);
+ defaultDuration == curLeaderElectionTimeout ?
+ RATIS_LEADER_ELECTION_TIMEOUT : curLeaderElectionTimeout,
+ TimeUnit.MILLISECONDS);
+ long defaultNodeFailureTimeout =
+ OMConfigKeys.OZONE_OM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_DEFAULT.
+ getDuration();
+ long curNodeFailureTimeout = conf.getTimeDuration(
+ OMConfigKeys.OZONE_OM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_KEY,
+ defaultNodeFailureTimeout,
+ OMConfigKeys.OZONE_OM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_DEFAULT.
+ getUnit());
conf.setTimeDuration(
OMConfigKeys.OZONE_OM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_KEY,
- NODE_FAILURE_TIMEOUT, TimeUnit.MILLISECONDS);
+ curNodeFailureTimeout == defaultNodeFailureTimeout ?
+ NODE_FAILURE_TIMEOUT : curNodeFailureTimeout,
+ TimeUnit.MILLISECONDS);
}
/**
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/RatisTestHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/RatisTestHelper.java
index b83a2c9..535ca91 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/RatisTestHelper.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/RatisTestHelper.java
@@ -23,7 +23,6 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.ratis.RatisHelper;
@@ -97,7 +96,7 @@
return conf;
}
- static void initRatisConf(RpcType rpc, Configuration conf) {
+ static void initRatisConf(RpcType rpc, OzoneConfiguration conf) {
conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY, true);
conf.set(OzoneConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY, rpc.name());
conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 1, TimeUnit.SECONDS);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
index c69aa71..8f8fe27 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
@@ -18,15 +18,6 @@
package org.apache.hadoop.ozone;
-import static org.apache.hadoop.hdds.protocol.DatanodeDetails.Port;
-import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
@@ -35,9 +26,6 @@
import java.util.HashSet;
import java.util.List;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.DFSConfigKeysLegacy;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -54,18 +42,27 @@
import org.apache.hadoop.ozone.container.ozoneimpl.TestOzoneContainer;
import org.apache.hadoop.test.PathUtils;
import org.apache.hadoop.test.TestGenericTestUtils;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.RandomUtils;
+import static org.apache.hadoop.hdds.protocol.DatanodeDetails.Port;
+import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
import org.yaml.snakeyaml.Yaml;
/**
* Test cases for mini ozone cluster.
*/
-@Ignore
public class TestMiniOzoneCluster {
private MiniOzoneCluster cluster;
@@ -192,7 +189,7 @@
@Test
public void testContainerRandomPort() throws IOException {
- Configuration ozoneConf = SCMTestUtils.getConf();
+ OzoneConfiguration ozoneConf = SCMTestUtils.getConf();
File testDir = PathUtils.getTestDir(TestOzoneContainer.class);
ozoneConf.set(DFSConfigKeysLegacy.DFS_DATANODE_DATA_DIR_KEY,
testDir.getAbsolutePath());
@@ -326,4 +323,22 @@
endpoint.getState());
}
}
+
+ /**
+ * Test that multiple datanode directories are created in MiniOzoneCluster.
+ * @throws Exception
+ */
+ @Test (timeout = 60000)
+ public void testMultipleDataDirs() throws Exception {
+ // Start a cluster with 3 DN
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(1)
+ .setNumDataVolumes(3)
+ .build();
+ cluster.waitForClusterToBeReady();
+
+ Assert.assertEquals(3, cluster.getHddsDatanodes().get(0)
+ .getDatanodeStateMachine().getContainer().getVolumeSet()
+ .getVolumesList().size());
+ }
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
index d7a8d20..437d4bc 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
@@ -19,6 +19,7 @@
import org.apache.hadoop.conf.TestConfigurationFieldsBase;
import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.ozone.recon.ReconServerConfigKeys;
@@ -38,13 +39,13 @@
new Class[] {OzoneConfigKeys.class, ScmConfigKeys.class,
OMConfigKeys.class, HddsConfigKeys.class,
ReconServerConfigKeys.class,
- S3GatewayConfigKeys.class
+ S3GatewayConfigKeys.class,
+ SCMHTTPServerConfig.class
};
errorIfMissingConfigProps = true;
errorIfMissingXmlProps = true;
xmlPropsToSkipCompare.add("hadoop.tags.custom");
xmlPropsToSkipCompare.add("ozone.om.nodes.EXAMPLEOMSERVICEID");
- xmlPrefixToSkipCompare.add("ipc.client.rpc-timeout.ms");
addPropertiesNotInXml();
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index 88ba220..f65c17f 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -34,7 +34,7 @@
import java.util.UUID;
import java.util.concurrent.Callable;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
@@ -605,7 +605,7 @@
// Setup secure OM for start.
OzoneConfiguration newConf = new OzoneConfiguration(conf);
- int tokenMaxLifetime = 500;
+ int tokenMaxLifetime = 1000;
newConf.setLong(DELEGATION_TOKEN_MAX_LIFETIME_KEY, tokenMaxLifetime);
setupOm(newConf);
long omVersion =
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java
index 9beddd4..e36bf73 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java
@@ -21,9 +21,11 @@
import java.util.Map;
import java.util.Set;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
+import org.apache.hadoop.hdds.utils.MetadataKeyFilters;
+import org.apache.hadoop.hdds.utils.MetadataKeyFilters.KeyPrefixFilter;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB;
@@ -32,8 +34,6 @@
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
-import org.apache.hadoop.hdds.utils.MetadataKeyFilters;
-import org.apache.hadoop.hdds.utils.MetadataKeyFilters.KeyPrefixFilter;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -48,10 +48,10 @@
public class TestStorageContainerManagerHelper {
private final MiniOzoneCluster cluster;
- private final Configuration conf;
+ private final OzoneConfiguration conf;
public TestStorageContainerManagerHelper(MiniOzoneCluster cluster,
- Configuration conf) throws IOException {
+ OzoneConfiguration conf) throws IOException {
this.cluster = cluster;
this.conf = conf;
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java
index d05093f..7664d75 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java
@@ -16,17 +16,6 @@
*/
package org.apache.hadoop.ozone.client;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
-import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest;
-import org.apache.hadoop.hdds.security.x509.certificates.utils.SelfSignedCertificate;
-import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
-import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
-import org.bouncycastle.cert.X509CertificateHolder;
-import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
-
import java.io.InputStream;
import java.security.KeyPair;
import java.security.PrivateKey;
@@ -37,6 +26,17 @@
import java.time.temporal.ChronoUnit;
import java.util.List;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
+import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest;
+import org.apache.hadoop.hdds.security.x509.certificates.utils.SelfSignedCertificate;
+import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
+import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
+
+import org.bouncycastle.cert.X509CertificateHolder;
+import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
+
/**
* Test implementation for CertificateClient. To be used only for test
* purposes.
@@ -46,7 +46,7 @@
private final SecurityConfig securityConfig;
private final KeyPair keyPair;
- private final Configuration config;
+ private final OzoneConfiguration config;
private final X509Certificate x509Certificate;
public CertificateClientTestImpl(OzoneConfiguration conf) throws Exception {
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/Test2BlockOutputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/Test2BlockOutputStream.java
new file mode 100644
index 0000000..3bb55fa
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/Test2BlockOutputStream.java
@@ -0,0 +1,173 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.client.rpc;
+
+import org.apache.hadoop.conf.StorageUnit;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.storage.BlockOutputStream;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.io.KeyOutputStream;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.ozone.container.TestHelper;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_DELAY;
+
+/**
+ * Tests BlockOutputStream class.
+ */
+public class Test2BlockOutputStream {
+ private static MiniOzoneCluster cluster;
+ private static OzoneConfiguration conf = new OzoneConfiguration();
+ private static OzoneClient client;
+ private static ObjectStore objectStore;
+ private static int chunkSize;
+ private static int flushSize;
+ private static int maxFlushSize;
+ private static int blockSize;
+ private static String volumeName;
+ private static String bucketName;
+ private static String keyString;
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ * <p>
+ * Ozone is made active by setting OZONE_ENABLED = true
+ *
+ * @throws IOException
+ */
+ @BeforeClass
+ public static void init() throws Exception {
+ chunkSize = 100;
+ flushSize = 2 * chunkSize;
+ maxFlushSize = 2 * flushSize;
+ blockSize = 2 * maxFlushSize;
+ conf.setTimeDuration(HDDS_SCM_WATCHER_TIMEOUT, 1000, TimeUnit.MILLISECONDS);
+ conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, TimeUnit.SECONDS);
+ conf.set(OzoneConfigKeys.OZONE_CLIENT_CHECKSUM_TYPE, "NONE");
+ conf.setQuietMode(false);
+ conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 4,
+ StorageUnit.MB);
+ conf.setBoolean(OZONE_CLIENT_STREAM_BUFFER_FLUSH_DELAY, true);
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(7)
+ .setTotalPipelineNumLimit(10)
+ .setBlockSize(blockSize)
+ .setChunkSize(chunkSize)
+ .setStreamBufferFlushSize(flushSize)
+ .setStreamBufferMaxSize(maxFlushSize)
+ .setStreamBufferSizeUnit(StorageUnit.BYTES)
+ .build();
+ cluster.waitForClusterToBeReady();
+ //the easiest way to create an open container is creating a key
+ client = OzoneClientFactory.getRpcClient(conf);
+ objectStore = client.getObjectStore();
+ keyString = UUID.randomUUID().toString();
+ volumeName = "testblockoutputstream";
+ bucketName = volumeName;
+ objectStore.createVolume(volumeName);
+ objectStore.getVolume(volumeName).createBucket(bucketName);
+ }
+
+ private String getKeyName() {
+ return UUID.randomUUID().toString();
+ }
+
+ /**
+ * Shutdown MiniDFSCluster.
+ */
+ @AfterClass
+ public static void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testFlushChunkDelay() throws Exception {
+ String keyName1 = getKeyName();
+ OzoneOutputStream key1 = createKey(keyName1, ReplicationType.RATIS, 0);
+
+ byte[] data1 =
+ ContainerTestHelper.getFixedLengthString(keyString, 10)
+ .getBytes(UTF_8);
+ key1.write(data1);
+ key1.flush();
+ KeyOutputStream keyOutputStream = (KeyOutputStream)key1.getOutputStream();
+ Assert.assertTrue(keyOutputStream.getStreamEntries().size() == 1);
+ OutputStream stream = keyOutputStream.getStreamEntries().get(0)
+ .getOutputStream();
+ Assert.assertTrue(stream instanceof BlockOutputStream);
+ BlockOutputStream blockOutputStream = (BlockOutputStream) stream;
+
+ // we have just written data(length 10) less than chunk Size,
+ // at this time we call flush will not sync data.
+ Assert.assertEquals(0, blockOutputStream.getTotalDataFlushedLength());
+ key1.close();
+ validateData(keyName1, data1);
+
+ String keyName2 = getKeyName();
+ OzoneOutputStream key2 = createKey(keyName2, ReplicationType.RATIS, 0);
+ byte[] data2 =
+ ContainerTestHelper.getFixedLengthString(keyString, 110)
+ .getBytes(UTF_8);
+ key2.write(data2);
+ key2.flush();
+ keyOutputStream = (KeyOutputStream)key2.getOutputStream();
+ Assert.assertTrue(keyOutputStream.getStreamEntries().size() == 1);
+ stream = keyOutputStream.getStreamEntries().get(0)
+ .getOutputStream();
+ Assert.assertTrue(stream instanceof BlockOutputStream);
+ blockOutputStream = (BlockOutputStream) stream;
+
+ // we have just written data(length 110) greater than chunk Size,
+ // at this time we call flush will sync data.
+ Assert.assertEquals(data2.length,
+ blockOutputStream.getTotalDataFlushedLength());
+ key2.close();
+ validateData(keyName2, data2);
+ }
+
+ private OzoneOutputStream createKey(String keyName, ReplicationType type,
+ long size) throws Exception {
+ return TestHelper
+ .createKey(keyName, type, size, objectStore, volumeName, bucketName);
+ }
+ private void validateData(String keyName, byte[] data) throws Exception {
+ TestHelper
+ .validateData(keyName, data, objectStore, volumeName, bucketName);
+ }
+
+}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java
index 8f2010f..9c8f4d7 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java
@@ -124,9 +124,9 @@
ContainerProtos.Type.WriteChunk);
long putBlockCount = metrics.getContainerOpCountMetrics(
ContainerProtos.Type.PutBlock);
- long pendingWriteChunkCount = metrics.getContainerOpsMetrics(
+ long pendingWriteChunkCount = metrics.getPendingContainerOpCountMetrics(
ContainerProtos.Type.WriteChunk);
- long pendingPutBlockCount = metrics.getContainerOpsMetrics(
+ long pendingPutBlockCount = metrics.getPendingContainerOpCountMetrics(
ContainerProtos.Type.PutBlock);
long totalOpCount = metrics.getTotalOpCount();
String keyName = getKeyName();
@@ -157,10 +157,11 @@
Assert.assertEquals(0, blockOutputStream.getTotalAckDataLength());
Assert.assertEquals(pendingWriteChunkCount,
XceiverClientManager.getXceiverClientMetrics()
- .getContainerOpsMetrics(ContainerProtos.Type.WriteChunk));
+ .getPendingContainerOpCountMetrics(
+ ContainerProtos.Type.WriteChunk));
Assert.assertEquals(pendingPutBlockCount,
XceiverClientManager.getXceiverClientMetrics()
- .getContainerOpsMetrics(ContainerProtos.Type.PutBlock));
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
// commitIndex2FlushedData Map will be empty here
Assert.assertTrue(
@@ -171,10 +172,10 @@
key.flush();
// flush is a sync call, all pending operations will complete
- Assert.assertEquals(pendingWriteChunkCount,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.WriteChunk));
- Assert.assertEquals(pendingPutBlockCount,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.PutBlock));
+ Assert.assertEquals(pendingWriteChunkCount, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
+ Assert.assertEquals(pendingPutBlockCount, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
// we have just written data less than a chunk size, the data will just sit
// in the buffer, with only one buffer being allocated in the buffer pool
@@ -194,10 +195,10 @@
// now close the stream, It will update the ack length after watchForCommit
key.close();
- Assert.assertEquals(pendingWriteChunkCount,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.WriteChunk));
- Assert.assertEquals(pendingPutBlockCount,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.PutBlock));
+ Assert.assertEquals(pendingWriteChunkCount, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
+ Assert.assertEquals(pendingPutBlockCount, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
Assert.assertEquals(writeChunkCount + 1,
metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
Assert.assertEquals(putBlockCount + 2,
@@ -222,9 +223,9 @@
ContainerProtos.Type.WriteChunk);
long putBlockCount = metrics.getContainerOpCountMetrics(
ContainerProtos.Type.PutBlock);
- long pendingWriteChunkCount = metrics.getContainerOpsMetrics(
+ long pendingWriteChunkCount = metrics.getPendingContainerOpCountMetrics(
ContainerProtos.Type.WriteChunk);
- long pendingPutBlockCount = metrics.getContainerOpsMetrics(
+ long pendingPutBlockCount = metrics.getPendingContainerOpCountMetrics(
ContainerProtos.Type.PutBlock);
long totalOpCount = metrics.getTotalOpCount();
String keyName = getKeyName();
@@ -235,10 +236,10 @@
ContainerTestHelper.getFixedLengthString(keyString, dataLength)
.getBytes(UTF_8);
key.write(data1);
- Assert.assertEquals(pendingWriteChunkCount + 2,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.WriteChunk));
- Assert.assertEquals(pendingPutBlockCount + 1,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.PutBlock));
+ Assert.assertEquals(pendingWriteChunkCount + 2, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
+ Assert.assertEquals(pendingPutBlockCount + 1, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
Assert.assertTrue(key.getOutputStream() instanceof KeyOutputStream);
KeyOutputStream keyOutputStream = (KeyOutputStream)key.getOutputStream();
@@ -267,10 +268,10 @@
key.flush();
Assert.assertEquals(1, keyOutputStream.getStreamEntries().size());
// flush is a sync call, all pending operations will complete
- Assert.assertEquals(pendingWriteChunkCount,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.WriteChunk));
- Assert.assertEquals(pendingPutBlockCount,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.PutBlock));
+ Assert.assertEquals(pendingWriteChunkCount, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
+ Assert.assertEquals(pendingPutBlockCount, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
// Since the data in the buffer is already flushed, flush here will have
// no impact on the counters and data structures
@@ -295,10 +296,10 @@
.assertEquals(0, blockOutputStream.getBufferPool().computeBufferData());
Assert.assertEquals(dataLength, blockOutputStream.getTotalAckDataLength());
Assert.assertNull(blockOutputStream.getCommitIndex2flushedDataMap());
- Assert.assertEquals(pendingWriteChunkCount,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.WriteChunk));
- Assert.assertEquals(pendingPutBlockCount,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.PutBlock));
+ Assert.assertEquals(pendingWriteChunkCount, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
+ Assert.assertEquals(pendingPutBlockCount, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
Assert.assertEquals(writeChunkCount + 2,
metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
Assert.assertEquals(putBlockCount + 2,
@@ -317,9 +318,9 @@
ContainerProtos.Type.WriteChunk);
long putBlockCount = metrics.getContainerOpCountMetrics(
ContainerProtos.Type.PutBlock);
- long pendingWriteChunkCount = metrics.getContainerOpsMetrics(
+ long pendingWriteChunkCount = metrics.getPendingContainerOpCountMetrics(
ContainerProtos.Type.WriteChunk);
- long pendingPutBlockCount = metrics.getContainerOpsMetrics(
+ long pendingPutBlockCount = metrics.getPendingContainerOpCountMetrics(
ContainerProtos.Type.PutBlock);
long totalOpCount = metrics.getTotalOpCount();
String keyName = getKeyName();
@@ -330,10 +331,10 @@
ContainerTestHelper.getFixedLengthString(keyString, dataLength)
.getBytes(UTF_8);
key.write(data1);
- Assert.assertEquals(pendingWriteChunkCount + 1,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.WriteChunk));
- Assert.assertEquals(pendingPutBlockCount,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.PutBlock));
+ Assert.assertEquals(pendingWriteChunkCount + 1, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
+ Assert.assertEquals(pendingPutBlockCount, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
Assert.assertTrue(key.getOutputStream() instanceof KeyOutputStream);
KeyOutputStream keyOutputStream = (KeyOutputStream)key.getOutputStream();
@@ -366,10 +367,10 @@
metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
Assert.assertEquals(putBlockCount + 1,
metrics.getContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
- Assert.assertEquals(pendingWriteChunkCount,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.WriteChunk));
- Assert.assertEquals(pendingPutBlockCount,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.PutBlock));
+ Assert.assertEquals(pendingWriteChunkCount, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
+ Assert.assertEquals(pendingPutBlockCount, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
Assert.assertEquals(2, blockOutputStream.getBufferPool().getSize());
Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());
@@ -390,10 +391,10 @@
.assertEquals(0, blockOutputStream.getBufferPool().computeBufferData());
Assert.assertEquals(dataLength, blockOutputStream.getTotalAckDataLength());
Assert.assertNull(blockOutputStream.getCommitIndex2flushedDataMap());
- Assert.assertEquals(pendingWriteChunkCount,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.WriteChunk));
- Assert.assertEquals(pendingPutBlockCount,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.PutBlock));
+ Assert.assertEquals(pendingWriteChunkCount, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
+ Assert.assertEquals(pendingPutBlockCount, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
Assert.assertEquals(writeChunkCount + 2,
metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
Assert.assertEquals(putBlockCount + 2,
@@ -412,9 +413,9 @@
ContainerProtos.Type.WriteChunk);
long putBlockCount = metrics.getContainerOpCountMetrics(
ContainerProtos.Type.PutBlock);
- long pendingWriteChunkCount = metrics.getContainerOpsMetrics(
+ long pendingWriteChunkCount = metrics.getPendingContainerOpCountMetrics(
ContainerProtos.Type.WriteChunk);
- long pendingPutBlockCount = metrics.getContainerOpsMetrics(
+ long pendingPutBlockCount = metrics.getPendingContainerOpCountMetrics(
ContainerProtos.Type.PutBlock);
long totalOpCount = metrics.getTotalOpCount();
String keyName = getKeyName();
@@ -425,10 +426,10 @@
ContainerTestHelper.getFixedLengthString(keyString, dataLength)
.getBytes(UTF_8);
key.write(data1);
- Assert.assertEquals(pendingWriteChunkCount + 2,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.WriteChunk));
- Assert.assertEquals(pendingPutBlockCount + 1,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.PutBlock));
+ Assert.assertEquals(pendingWriteChunkCount + 2, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
+ Assert.assertEquals(pendingPutBlockCount + 1, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
Assert.assertTrue(key.getOutputStream() instanceof KeyOutputStream);
KeyOutputStream keyOutputStream = (KeyOutputStream)key.getOutputStream();
@@ -460,10 +461,10 @@
Assert.assertEquals(0, blockOutputStream.getTotalAckDataLength());
Assert.assertEquals(1, keyOutputStream.getStreamEntries().size());
key.close();
- Assert.assertEquals(pendingWriteChunkCount,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.WriteChunk));
- Assert.assertEquals(pendingPutBlockCount,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.PutBlock));
+ Assert.assertEquals(pendingWriteChunkCount, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
+ Assert.assertEquals(pendingPutBlockCount, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
Assert.assertEquals(writeChunkCount + 3,
metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
Assert.assertEquals(putBlockCount + 2,
@@ -488,9 +489,9 @@
ContainerProtos.Type.WriteChunk);
long putBlockCount = metrics.getContainerOpCountMetrics(
ContainerProtos.Type.PutBlock);
- long pendingWriteChunkCount = metrics.getContainerOpsMetrics(
+ long pendingWriteChunkCount = metrics.getPendingContainerOpCountMetrics(
ContainerProtos.Type.WriteChunk);
- long pendingPutBlockCount = metrics.getContainerOpsMetrics(
+ long pendingPutBlockCount = metrics.getPendingContainerOpCountMetrics(
ContainerProtos.Type.PutBlock);
long totalOpCount = metrics.getTotalOpCount();
String keyName = getKeyName();
@@ -504,11 +505,11 @@
// since its hitting the full bufferCondition, it will call watchForCommit
// and completes atleast putBlock for first flushSize worth of data
+ Assert.assertTrue(metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk)
+ <= pendingWriteChunkCount + 2);
Assert.assertTrue(
- metrics.getContainerOpsMetrics(ContainerProtos.Type.WriteChunk)
- <= pendingWriteChunkCount + 2);
- Assert.assertTrue(
- metrics.getContainerOpsMetrics(ContainerProtos.Type.PutBlock)
+ metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock)
<= pendingPutBlockCount + 1);
Assert.assertTrue(key.getOutputStream() instanceof KeyOutputStream);
KeyOutputStream keyOutputStream = (KeyOutputStream)key.getOutputStream();
@@ -542,10 +543,10 @@
// the map.
key.flush();
Assert.assertEquals(1, keyOutputStream.getStreamEntries().size());
- Assert.assertEquals(pendingWriteChunkCount,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.WriteChunk));
- Assert.assertEquals(pendingPutBlockCount,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.PutBlock));
+ Assert.assertEquals(pendingWriteChunkCount, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
+ Assert.assertEquals(pendingPutBlockCount, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
// Since the data in the buffer is already flushed, flush here will have
// no impact on the counters and data structures
@@ -560,10 +561,10 @@
// now close the stream, It will update the ack length after watchForCommit
key.close();
- Assert.assertEquals(pendingWriteChunkCount,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.WriteChunk));
- Assert.assertEquals(pendingPutBlockCount,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.PutBlock));
+ Assert.assertEquals(pendingWriteChunkCount, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
+ Assert.assertEquals(pendingPutBlockCount, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
Assert.assertEquals(writeChunkCount + 4,
metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
Assert.assertEquals(putBlockCount + 3,
@@ -588,9 +589,9 @@
ContainerProtos.Type.WriteChunk);
long putBlockCount = metrics.getContainerOpCountMetrics(
ContainerProtos.Type.PutBlock);
- long pendingWriteChunkCount = metrics.getContainerOpsMetrics(
+ long pendingWriteChunkCount = metrics.getPendingContainerOpCountMetrics(
ContainerProtos.Type.WriteChunk);
- long pendingPutBlockCount = metrics.getContainerOpsMetrics(
+ long pendingPutBlockCount = metrics.getPendingContainerOpCountMetrics(
ContainerProtos.Type.PutBlock);
long totalOpCount = metrics.getTotalOpCount();
String keyName = getKeyName();
@@ -606,11 +607,11 @@
// since its hitting the full bufferCondition, it will call watchForCommit
// and completes atleast putBlock for first flushSize worth of data
+ Assert.assertTrue(metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk)
+ <= pendingWriteChunkCount + 2);
Assert.assertTrue(
- metrics.getContainerOpsMetrics(ContainerProtos.Type.WriteChunk)
- <= pendingWriteChunkCount + 2);
- Assert.assertTrue(
- metrics.getContainerOpsMetrics(ContainerProtos.Type.PutBlock)
+ metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock)
<= pendingPutBlockCount + 1);
Assert.assertEquals(writeChunkCount + 4,
metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
@@ -645,10 +646,10 @@
// the map.
key.flush();
Assert.assertEquals(1, keyOutputStream.getStreamEntries().size());
- Assert.assertEquals(pendingWriteChunkCount,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.WriteChunk));
- Assert.assertEquals(pendingPutBlockCount,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.PutBlock));
+ Assert.assertEquals(pendingWriteChunkCount, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
+ Assert.assertEquals(pendingPutBlockCount, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
// Since the data in the buffer is already flushed, flush here will have
// no impact on the counters and data structures
@@ -668,10 +669,10 @@
// make sure the bufferPool is empty
Assert
.assertEquals(0, blockOutputStream.getBufferPool().computeBufferData());
- Assert.assertEquals(pendingWriteChunkCount,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.WriteChunk));
- Assert.assertEquals(pendingPutBlockCount,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.PutBlock));
+ Assert.assertEquals(pendingWriteChunkCount, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
+ Assert.assertEquals(pendingPutBlockCount, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
Assert.assertEquals(writeChunkCount + 5,
metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
Assert.assertEquals(putBlockCount + 4,
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDeleteWithSlowFollower.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDeleteWithSlowFollower.java
index 9cabeb3..0daae95 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDeleteWithSlowFollower.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDeleteWithSlowFollower.java
@@ -17,9 +17,16 @@
package org.apache.hadoop.ozone.client.rpc;
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -37,7 +44,6 @@
import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.io.KeyOutputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
@@ -51,27 +57,23 @@
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_COMMAND_STATUS_REPORT_INTERVAL;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
/**
* Tests delete key operation with a slow follower in the datanode
* pipeline.
*/
+@Ignore("HDDS-3330")
public class TestDeleteWithSlowFollower {
private static MiniOzoneCluster cluster;
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestKeyInputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestKeyInputStream.java
index 2779e7f..d8c4b3b 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestKeyInputStream.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestKeyInputStream.java
@@ -36,7 +36,10 @@
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.junit.Rule;
+import org.junit.rules.Timeout;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Random;
import java.util.UUID;
@@ -100,6 +103,9 @@
objectStore.getVolume(volumeName).createBucket(bucketName);
}
+ @Rule
+ public Timeout timeout = new Timeout(300_000);
+
/**
* Shutdown MiniDFSCluster.
*/
@@ -114,18 +120,9 @@
return UUID.randomUUID().toString();
}
- private OzoneOutputStream createKey(String keyName, ReplicationType type,
- long size) throws Exception {
- return TestHelper
- .createKey(keyName, type, size, objectStore, volumeName, bucketName);
- }
-
@Test
public void testSeekRandomly() throws Exception {
- XceiverClientMetrics metrics = XceiverClientManager
- .getXceiverClientMetrics();
-
String keyName = getKeyName();
OzoneOutputStream key = TestHelper.createKey(keyName,
ReplicationType.RATIS, 0, objectStore, volumeName, bucketName);
@@ -223,6 +220,7 @@
@Test
public void testSeek() throws Exception {
+ XceiverClientManager.resetXceiverClientMetrics();
XceiverClientMetrics metrics = XceiverClientManager
.getXceiverClientMetrics();
long writeChunkCount = metrics.getContainerOpCountMetrics(
@@ -255,7 +253,7 @@
// Seek operation should not result in any readChunk operation.
Assert.assertEquals(readChunkCount, metrics
- .getContainerOpsMetrics(ContainerProtos.Type.ReadChunk));
+ .getContainerOpCountMetrics(ContainerProtos.Type.ReadChunk));
byte[] readData = new byte[chunkSize];
keyInputStream.read(readData, 0, chunkSize);
@@ -273,4 +271,64 @@
Assert.assertEquals(inputData[chunkSize + 50 + i], readData[i]);
}
}
+
+ @Test
+ public void testCopyLarge() throws Exception {
+ String keyName = getKeyName();
+ OzoneOutputStream key = TestHelper.createKey(keyName,
+ ReplicationType.RATIS, 0, objectStore, volumeName, bucketName);
+
+ // write data spanning 3 blocks
+ int dataLength = (2 * blockSize) + (blockSize / 2);
+
+ byte[] inputData = new byte[dataLength];
+ Random rand = new Random();
+ for (int i = 0; i < dataLength; i++) {
+ inputData[i] = (byte) rand.nextInt(127);
+ }
+ key.write(inputData);
+ key.close();
+
+ // test with random start and random length
+ for (int i = 0; i < 100; i++) {
+ int inputOffset = rand.nextInt(dataLength - 1);
+ int length = rand.nextInt(dataLength - inputOffset);
+
+ KeyInputStream keyInputStream = (KeyInputStream) objectStore
+ .getVolume(volumeName).getBucket(bucketName).readKey(keyName)
+ .getInputStream();
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+
+ keyInputStream.copyLarge(outputStream, inputOffset, length,
+ new byte[4096]);
+ byte[] readData = outputStream.toByteArray();
+ keyInputStream.close();
+ outputStream.close();
+
+ for (int j = inputOffset; j < inputOffset + length; j++) {
+ Assert.assertEquals(readData[j - inputOffset], inputData[j]);
+ }
+ }
+
+ // test with random start and -ve length
+ for (int i = 0; i < 10; i++) {
+ int inputOffset = rand.nextInt(dataLength - 1);
+ int length = -1;
+
+ KeyInputStream keyInputStream = (KeyInputStream) objectStore
+ .getVolume(volumeName).getBucket(bucketName).readKey(keyName)
+ .getInputStream();
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+
+ keyInputStream.copyLarge(outputStream, inputOffset, length,
+ new byte[4096]);
+ byte[] readData = outputStream.toByteArray();
+ keyInputStream.close();
+ outputStream.close();
+
+ for (int j = inputOffset; j < dataLength; j++) {
+ Assert.assertEquals(readData[j - inputOffset], inputData[j]);
+ }
+ }
+ }
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java
index 3f4b19a..324db98 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java
@@ -17,7 +17,14 @@
*/
package org.apache.hadoop.ozone.client.rpc;
-import org.apache.hadoop.conf.Configuration;
+import java.io.File;
+import java.io.IOException;
+import java.security.NoSuchAlgorithmException;
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
import org.apache.hadoop.crypto.key.kms.server.MiniKMS;
@@ -49,22 +56,14 @@
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.test.GenericTestUtils;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
-import java.io.File;
-import java.io.IOException;
-import java.security.NoSuchAlgorithmException;
-import java.time.Instant;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.UUID;
-
-import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
-
/**
* This class is to test all the public facing APIs of Ozone Client.
*/
@@ -312,7 +311,7 @@
}
private static void createKey(String keyName, KeyProvider
- provider, Configuration config)
+ provider, OzoneConfiguration config)
throws NoSuchAlgorithmException, IOException {
final KeyProvider.Options options = KeyProvider.options(config);
options.setDescription(keyName);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
index 030527a..f8c421a 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
@@ -34,8 +34,6 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.client.OzoneQuota;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
@@ -44,6 +42,7 @@
import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
@@ -69,6 +68,7 @@
import org.apache.hadoop.ozone.client.VolumeArgs;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.ozone.common.OzoneChecksumException;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
@@ -88,7 +88,6 @@
import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
-import org.apache.hadoop.ozone.s3.util.OzoneS3Util;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType;
import org.apache.hadoop.ozone.security.acl.OzoneAclConfig;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
@@ -101,7 +100,7 @@
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.RandomUtils;
-
+import org.apache.commons.lang3.StringUtils;
import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE;
import static org.apache.hadoop.hdds.client.ReplicationType.STAND_ALONE;
import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS;
@@ -109,23 +108,16 @@
import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType.GROUP;
import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType.USER;
import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.READ;
-import static org.hamcrest.CoreMatchers.containsString;
-import static org.hamcrest.CoreMatchers.either;
import org.junit.Assert;
-
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-
import org.junit.Ignore;
import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* This is an abstract class to test all the public facing APIs of Ozone
@@ -136,8 +128,6 @@
*/
public abstract class TestOzoneRpcClientAbstract {
- static final Logger LOG =
- LoggerFactory.getLogger(TestOzoneRpcClientAbstract.class);
private static MiniOzoneCluster cluster = null;
private static OzoneClient ozClient = null;
private static ObjectStore store = null;
@@ -171,6 +161,8 @@
cluster.waitForClusterToBeReady();
ozClient = OzoneClientFactory.getRpcClient(conf);
store = ozClient.getObjectStore();
+ String volumeName = HddsClientUtils.getS3VolumeName(conf);
+ store.createVolume(volumeName);
storageContainerLocationClient =
cluster.getStorageContainerLocationClient();
ozoneManager = cluster.getOzoneManager();
@@ -242,6 +234,19 @@
}
@Test
+ public void testVolumeSetOwner() throws IOException {
+ String volumeName = UUID.randomUUID().toString();
+ store.createVolume(volumeName);
+
+ String ownerName = "someRandomUser1";
+
+ ClientProtocol proxy = store.getClientProxy();
+ proxy.setVolumeOwner(volumeName, ownerName);
+ // Set owner again
+ proxy.setVolumeOwner(volumeName, ownerName);
+ }
+
+ @Test
public void testSetVolumeQuota()
throws IOException {
String volumeName = UUID.randomUUID().toString();
@@ -316,80 +321,26 @@
public void testCreateS3Bucket()
throws IOException {
Instant testStartTime = Instant.now();
- String userName = UserGroupInformation.getCurrentUser().getUserName();
String bucketName = UUID.randomUUID().toString();
- store.createS3Bucket(userName, bucketName);
- String volumeName = store.getOzoneVolumeName(bucketName);
- OzoneVolume volume = store.getVolume(volumeName);
- OzoneBucket bucket = volume.getBucket(bucketName);
+ store.createS3Bucket(bucketName);
+ OzoneBucket bucket = store.getS3Bucket(bucketName);
Assert.assertEquals(bucketName, bucket.getName());
Assert.assertFalse(bucket.getCreationTime().isBefore(testStartTime));
- Assert.assertFalse(volume.getCreationTime().isBefore(testStartTime));
- }
-
- @Test
- public void testCreateSecureS3Bucket() throws IOException {
- Instant testStartTime = Instant.now();
- String userName = "ozone/localhost@EXAMPLE.COM";
- String bucketName = UUID.randomUUID().toString();
- String s3VolumeName = OzoneS3Util.getS3Username(userName);
- store.createS3Bucket(s3VolumeName, bucketName);
- String volumeName = store.getOzoneVolumeName(bucketName);
- assertEquals(volumeName, "s3" + s3VolumeName);
-
- OzoneVolume volume = store.getVolume(volumeName);
- OzoneBucket bucket = volume.getBucket(bucketName);
- Assert.assertEquals(bucketName, bucket.getName());
- Assert.assertFalse(bucket.getCreationTime().isBefore(testStartTime));
- Assert.assertFalse(volume.getCreationTime().isBefore(testStartTime));
- }
-
-
- @Test
- public void testListS3Buckets()
- throws IOException {
- String userName = "ozone100";
- String bucketName1 = UUID.randomUUID().toString();
- String bucketName2 = UUID.randomUUID().toString();
- store.createS3Bucket(userName, bucketName1);
- store.createS3Bucket(userName, bucketName2);
- Iterator<? extends OzoneBucket> iterator = store.listS3Buckets(userName,
- null);
-
- while (iterator.hasNext()) {
- assertThat(iterator.next().getName(), either(containsString(bucketName1))
- .or(containsString(bucketName2)));
- }
-
- }
-
- @Test
- public void testListS3BucketsFail() {
- String userName = "randomUser";
- Iterator<? extends OzoneBucket> iterator = store.listS3Buckets(userName,
- null);
-
- Assert.assertFalse(iterator.hasNext());
-
}
@Test
public void testDeleteS3Bucket()
throws Exception {
Instant testStartTime = Instant.now();
- String userName = "ozone1";
String bucketName = UUID.randomUUID().toString();
- store.createS3Bucket(userName, bucketName);
- String volumeName = store.getOzoneVolumeName(bucketName);
- OzoneVolume volume = store.getVolume(volumeName);
- OzoneBucket bucket = volume.getBucket(bucketName);
+ store.createS3Bucket(bucketName);
+ OzoneBucket bucket = store.getS3Bucket(bucketName);
Assert.assertEquals(bucketName, bucket.getName());
Assert.assertFalse(bucket.getCreationTime().isBefore(testStartTime));
- Assert.assertFalse(volume.getCreationTime().isBefore(testStartTime));
store.deleteS3Bucket(bucketName);
- OzoneTestUtils.expectOmException(ResultCodes.S3_BUCKET_NOT_FOUND,
- () -> store.getOzoneVolumeName(bucketName));
+ OzoneTestUtils.expectOmException(ResultCodes.BUCKET_NOT_FOUND,
+ () -> store.getS3Bucket(bucketName));
}
@Test
@@ -402,24 +353,6 @@
}
@Test
- public void testCreateS3BucketMapping()
- throws IOException {
- String userName = OzoneConsts.OZONE;
- String bucketName = UUID.randomUUID().toString();
- store.createS3Bucket(userName, bucketName);
- String volumeName = store.getOzoneVolumeName(bucketName);
- OzoneVolume volume = store.getVolume(volumeName);
- OzoneBucket bucket = volume.getBucket(bucketName);
- Assert.assertEquals(bucketName, bucket.getName());
-
- String mapping = store.getOzoneBucketMapping(bucketName);
- Assert.assertEquals("s3"+userName+"/"+bucketName, mapping);
- Assert.assertEquals(bucketName, store.getOzoneBucketName(bucketName));
- Assert.assertEquals("s3"+userName, store.getOzoneVolumeName(bucketName));
-
- }
-
- @Test
public void testCreateBucketWithVersioning()
throws IOException, OzoneClientException {
String volumeName = UUID.randomUUID().toString();
@@ -498,7 +431,7 @@
String bucketName = "invalid#bucket";
store.createVolume(volumeName);
OzoneVolume volume = store.getVolume(volumeName);
- LambdaTestUtils.intercept(IllegalArgumentException.class,
+ LambdaTestUtils.intercept(OMException.class,
"Bucket or Volume name has an unsupported" +
" character : #",
() -> volume.createBucket(bucketName));
@@ -942,7 +875,7 @@
private void readCorruptedKey(String volumeName, String bucketName,
String keyName, boolean verifyChecksum) {
try {
- Configuration configuration = cluster.getConf();
+ OzoneConfiguration configuration = cluster.getConf();
configuration.setBoolean(OzoneConfigKeys.OZONE_CLIENT_VERIFY_CHECKSUM,
verifyChecksum);
RpcClient client = new RpcClient(configuration, null);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java
index 2f37c4a..b09042f 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java
@@ -25,6 +25,7 @@
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.security.token.BlockTokenVerifier;
@@ -117,6 +118,8 @@
cluster.waitForClusterToBeReady();
ozClient = OzoneClientFactory.getRpcClient(conf);
store = ozClient.getObjectStore();
+ String volumeName = HddsClientUtils.getS3VolumeName(conf);
+ store.createVolume(volumeName);
storageContainerLocationClient =
cluster.getStorageContainerLocationClient();
ozoneManager = cluster.getOzoneManager();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java
index 01dab0e..3e93b24 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java
@@ -152,9 +152,9 @@
ContainerProtos.Type.WriteChunk);
long putBlockCount = metrics.getContainerOpCountMetrics(
ContainerProtos.Type.PutBlock);
- long pendingWriteChunkCount = metrics.getContainerOpsMetrics(
+ long pendingWriteChunkCount = metrics.getPendingContainerOpCountMetrics(
ContainerProtos.Type.WriteChunk);
- long pendingPutBlockCount = metrics.getContainerOpsMetrics(
+ long pendingPutBlockCount = metrics.getPendingContainerOpCountMetrics(
ContainerProtos.Type.PutBlock);
long totalOpCount = metrics.getTotalOpCount();
String keyName = getKeyName();
@@ -167,11 +167,11 @@
key.write(data1);
// since its hitting the full bufferCondition, it will call watchForCommit
// and completes atleast putBlock for first flushSize worth of data
+ Assert.assertTrue(metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk)
+ <= pendingWriteChunkCount + 2);
Assert.assertTrue(
- metrics.getContainerOpsMetrics(ContainerProtos.Type.WriteChunk)
- <= pendingWriteChunkCount + 2);
- Assert.assertTrue(
- metrics.getContainerOpsMetrics(ContainerProtos.Type.PutBlock)
+ metrics.getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock)
<= pendingPutBlockCount + 1);
Assert.assertEquals(writeChunkCount + 4,
metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
@@ -205,10 +205,10 @@
// Now do a flush. This will flush the data and update the flush length and
// the map.
key.flush();
- Assert.assertEquals(pendingWriteChunkCount,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.WriteChunk));
- Assert.assertEquals(pendingPutBlockCount,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.PutBlock));
+ Assert.assertEquals(pendingWriteChunkCount, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
+ Assert.assertEquals(pendingPutBlockCount, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
Assert.assertEquals(writeChunkCount + 5,
metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
Assert.assertEquals(putBlockCount + 3,
@@ -253,10 +253,10 @@
.assertEquals(dataLength, blockOutputStream.getTotalAckDataLength());
Assert.assertNull(blockOutputStream.getCommitIndex2flushedDataMap());
Assert.assertEquals(0, keyOutputStream.getStreamEntries().size());
- Assert.assertEquals(pendingWriteChunkCount,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.WriteChunk));
- Assert.assertEquals(pendingPutBlockCount,
- metrics.getContainerOpsMetrics(ContainerProtos.Type.PutBlock));
+ Assert.assertEquals(pendingWriteChunkCount, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
+ Assert.assertEquals(pendingPutBlockCount, metrics
+ .getPendingContainerOpCountMetrics(ContainerProtos.Type.PutBlock));
Assert.assertEquals(writeChunkCount + 14,
metrics.getContainerOpCountMetrics(ContainerProtos.Type.WriteChunk));
Assert.assertEquals(putBlockCount + 8,
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/ratis/TestDnRatisLogParser.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/ratis/TestDnRatisLogParser.java
index cd3882b..f7eaee6 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/ratis/TestDnRatisLogParser.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/ratis/TestDnRatisLogParser.java
@@ -17,21 +17,22 @@
*/
package org.apache.hadoop.ozone.dn.ratis;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.segmentparser.DatanodeRatisLogParser;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.PrintStream;
import java.util.UUID;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.segmentparser.DatanodeRatisLogParser;
+
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
/**
* Test Datanode Ratis log parser.
*/
@@ -62,9 +63,9 @@
}
@Test
- public void testRatisLogParsing() {
+ public void testRatisLogParsing() throws Exception {
cluster.stop();
- Configuration conf = cluster.getHddsDatanodes().get(0).getConf();
+ OzoneConfiguration conf = cluster.getHddsDatanodes().get(0).getConf();
String path =
conf.get(OzoneConfigKeys.DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR);
UUID pid = cluster.getStorageContainerManager().getPipelineManager()
@@ -72,7 +73,7 @@
File pipelineDir = new File(path, pid.toString());
File currentDir = new File(pipelineDir, "current");
File logFile = new File(currentDir, "log_inprogress_0");
- Assert.assertTrue(logFile.exists());
+ GenericTestUtils.waitFor(logFile::exists, 100, 15000);
Assert.assertTrue(logFile.isFile());
DatanodeRatisLogParser datanodeRatisLogParser =
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
index 6e23f34..682df24 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
@@ -33,10 +33,6 @@
import java.util.TreeSet;
import java.util.UUID;
-import com.google.common.base.Optional;
-import com.google.common.collect.Sets;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.client.BlockID;
@@ -46,6 +42,7 @@
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.scm.TestUtils;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.MockNodeManager;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
@@ -90,8 +87,22 @@
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.LambdaTestUtils;
-
import org.apache.hadoop.util.Time;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.Sets;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.LEAF_SCHEMA;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.RACK_SCHEMA;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT_SCHEMA;
+import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_BLOCKS_MAX;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.SCM_GET_PIPELINE_EXCEPTION;
+import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.ALL;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -101,21 +112,8 @@
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
+import static org.mockito.Matchers.anyList;
import org.mockito.Mockito;
-
-import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_BLOCKS_MAX;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT;
-
-import static org.apache.hadoop.hdds.scm.net.NetConstants.LEAF_SCHEMA;
-import static org.apache.hadoop.hdds.scm.net.NetConstants.RACK_SCHEMA;
-import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT_SCHEMA;
-
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
-import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.SCM_GET_PIPELINE_EXCEPTION;
-import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.ALL;
-import static org.mockito.Matchers.anyLong;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
@@ -205,11 +203,11 @@
for (OzoneFileStatus fileStatus : fileStatuses) {
if (fileStatus.isFile()) {
keyManager.deleteKey(
- createKeyArgs(fileStatus.getPath().toString().substring(1)));
+ createKeyArgs(fileStatus.getKeyInfo().getKeyName()));
} else {
keyManager.deleteKey(createKeyArgs(OzoneFSUtils
.addTrailingSlashIfNeeded(
- fileStatus.getPath().toString().substring(1))));
+ fileStatus.getKeyInfo().getKeyName())));
}
}
}
@@ -1078,8 +1076,9 @@
Set<OzoneFileStatus> tmpStatusSet = new HashSet<>();
do {
tempFileStatus = keyManager.listStatus(dirArgs, false,
- tempFileStatus != null ? OzoneFSUtils.pathToKey(
- tempFileStatus.get(tempFileStatus.size() - 1).getPath()) : null,
+ tempFileStatus != null ?
+ tempFileStatus.get(tempFileStatus.size() - 1).getKeyInfo()
+ .getKeyName() : null,
2);
tmpStatusSet.addAll(tempFileStatus);
} while (tempFileStatus.size() == 2);
@@ -1094,8 +1093,10 @@
tmpStatusSet = new HashSet<>();
do {
tempFileStatus = keyManager.listStatus(dirArgs, true,
- tempFileStatus != null ? OzoneFSUtils.pathToKey(
- tempFileStatus.get(tempFileStatus.size() - 1).getPath()) : null,
+ tempFileStatus != null ?
+ tempFileStatus.get(tempFileStatus.size() - 1).getKeyInfo()
+ .getKeyName() :
+ null,
2);
tmpStatusSet.addAll(tempFileStatus);
} while (tempFileStatus.size() == 2);
@@ -1114,12 +1115,27 @@
StorageContainerLocationProtocol sclProtocolMock = mock(
StorageContainerLocationProtocol.class);
- ContainerWithPipeline containerWithPipelineMock =
- mock(ContainerWithPipeline.class);
- when(containerWithPipelineMock.getPipeline())
- .thenReturn(getRandomPipeline());
- when(sclProtocolMock.getContainerWithPipeline(anyLong()))
- .thenReturn(containerWithPipelineMock);
+
+ List<Long> containerIDs = new ArrayList<>();
+ containerIDs.add(100L);
+ containerIDs.add(200L);
+
+ List<ContainerWithPipeline> cps = new ArrayList<>();
+ for (Long containerID : containerIDs) {
+ ContainerWithPipeline containerWithPipelineMock =
+ mock(ContainerWithPipeline.class);
+ when(containerWithPipelineMock.getPipeline())
+ .thenReturn(getRandomPipeline());
+
+ ContainerInfo ci = mock(ContainerInfo.class);
+ when(ci.getContainerID()).thenReturn(containerID);
+ when(containerWithPipelineMock.getContainerInfo()).thenReturn(ci);
+
+ cps.add(containerWithPipelineMock);
+ }
+
+ when(sclProtocolMock.getContainerWithPipelineBatch(containerIDs))
+ .thenReturn(cps);
ScmClient scmClientMock = mock(ScmClient.class);
when(scmClientMock.getContainerClient()).thenReturn(sclProtocolMock);
@@ -1158,9 +1174,8 @@
keyManagerImpl.refreshPipeline(omKeyInfo);
- verify(sclProtocolMock, times(2)).getContainerWithPipeline(anyLong());
- verify(sclProtocolMock, times(1)).getContainerWithPipeline(100L);
- verify(sclProtocolMock, times(1)).getContainerWithPipeline(200L);
+ verify(sclProtocolMock, times(1))
+ .getContainerWithPipelineBatch(containerIDs);
} finally {
cluster.shutdown();
}
@@ -1178,7 +1193,7 @@
StorageContainerLocationProtocol sclProtocolMock = mock(
StorageContainerLocationProtocol.class);
doThrow(new IOException(errorMessage)).when(sclProtocolMock)
- .getContainerWithPipeline(anyLong());
+ .getContainerWithPipelineBatch(anyList());
ScmClient scmClientMock = mock(ScmClient.class);
when(scmClientMock.getContainerClient()).thenReturn(sclProtocolMock);
@@ -1263,8 +1278,10 @@
Set<String> fileSet, boolean recursive) {
for (OzoneFileStatus fileStatus : fileStatuses) {
- String keyName = OzoneFSUtils.pathToKey(fileStatus.getPath());
- String parent = Paths.get(keyName).getParent().toString();
+ String normalizedKeyName = fileStatus.getTrimmedName();
+ String parent =
+ Paths.get(fileStatus.getKeyInfo().getKeyName()).getParent()
+ .toString();
if (!recursive) {
// if recursive is false, verify all the statuses have the input
// directory as parent
@@ -1272,9 +1289,13 @@
}
// verify filestatus is present in directory or file set accordingly
if (fileStatus.isDirectory()) {
- Assert.assertTrue(directorySet.contains(keyName));
+ Assert
+ .assertTrue(directorySet + " doesn't contain " + normalizedKeyName,
+ directorySet.contains(normalizedKeyName));
} else {
- Assert.assertTrue(fileSet.contains(keyName));
+ Assert
+ .assertTrue(fileSet + " doesn't contain " + normalizedKeyName,
+ fileSet.contains(normalizedKeyName));
}
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java
index 4d6d86c..d21ebb4 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java
@@ -18,36 +18,41 @@
package org.apache.hadoop.ozone.om;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.
- OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.UUID;
-
import javax.servlet.ServletContext;
import javax.servlet.ServletException;
import javax.servlet.ServletOutputStream;
import javax.servlet.WriteListener;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.UUID;
-import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConsts;
+
+import org.apache.commons.io.FileUtils;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_FLUSH;
+import static org.apache.hadoop.ozone.om.OMDBCheckpointServlet.writeOmDBCheckpointToStream;
import org.junit.After;
import org.junit.Assert;
+import static org.junit.Assert.assertNotNull;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import org.junit.rules.Timeout;
import org.mockito.Matchers;
-
-import static org.apache.hadoop.ozone.OzoneConsts.
- OZONE_DB_CHECKPOINT_REQUEST_FLUSH;
import static org.mockito.Mockito.doCallRealMethod;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.mock;
@@ -67,6 +72,8 @@
@Rule
public Timeout timeout = new Timeout(60000);
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
/**
* Create a MiniDFSCluster for testing.
* <p>
@@ -166,4 +173,87 @@
}
}
+
+ @Test
+ public void testWriteCheckpointToOutputStream() throws Exception {
+
+ FileInputStream fis = null;
+ FileOutputStream fos = null;
+
+ try {
+ String testDirName = folder.newFolder().getAbsolutePath();
+ File file = new File(testDirName + "/temp1.txt");
+ FileWriter writer = new FileWriter(file);
+ writer.write("Test data 1");
+ writer.close();
+
+ file = new File(testDirName + "/temp2.txt");
+ writer = new FileWriter(file);
+ writer.write("Test data 2");
+ writer.close();
+
+ File outputFile =
+ new File(Paths.get(testDirName, "output_file.tgz").toString());
+ TestDBCheckpoint dbCheckpoint = new TestDBCheckpoint(
+ Paths.get(testDirName));
+ writeOmDBCheckpointToStream(dbCheckpoint,
+ new FileOutputStream(outputFile));
+ assertNotNull(outputFile);
+ } finally {
+ IOUtils.closeStream(fis);
+ IOUtils.closeStream(fos);
+ }
+ }
}
+
+class TestDBCheckpoint implements DBCheckpoint {
+
+ private Path checkpointFile;
+
+ TestDBCheckpoint(Path checkpointFile) {
+ this.checkpointFile = checkpointFile;
+ }
+
+ @Override
+ public Path getCheckpointLocation() {
+ return checkpointFile;
+ }
+
+ @Override
+ public long getCheckpointTimestamp() {
+ return 0;
+ }
+
+ @Override
+ public long getLatestSequenceNumber() {
+ return 0;
+ }
+
+ @Override
+ public long checkpointCreationTimeTaken() {
+ return 0;
+ }
+
+ @Override
+ public void cleanupCheckpoint() throws IOException {
+ FileUtils.deleteDirectory(checkpointFile.toFile());
+ }
+
+ @Override
+ public void setRatisSnapshotIndex(long omRatisSnapshotIndex) {
+ }
+
+ @Override
+ public long getRatisSnapshotIndex() {
+ return 0;
+ }
+
+ @Override
+ public void setRatisSnapshotTerm(long omRatisSnapshotTermIndex) {
+ }
+
+ @Override
+ public long getRatisSnapshotTerm() {
+ return 0;
+ }
+}
\ No newline at end of file
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
index 2091848..3712b71 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
@@ -34,10 +34,8 @@
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
-import org.apache.hadoop.test.MetricsAsserts;
import org.junit.After;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Test;
import org.mockito.Mockito;
@@ -157,23 +155,12 @@
}
@Test
- @Ignore("Test failing because of table cache. Revisit later.")
public void testBucketOps() throws IOException {
BucketManager bucketManager =
(BucketManager) HddsWhiteboxTestUtils.getInternalState(
ozoneManager, "bucketManager");
BucketManager mockBm = Mockito.spy(bucketManager);
- S3BucketManager s3BucketManager =
- (S3BucketManager) HddsWhiteboxTestUtils.getInternalState(
- ozoneManager, "s3BucketManager");
- S3BucketManager mockS3Bm = Mockito.spy(s3BucketManager);
-
- Mockito.doNothing().when(mockS3Bm).createS3Bucket("random", "random");
- Mockito.doNothing().when(mockS3Bm).deleteS3Bucket("random");
- Mockito.doReturn(true).when(mockS3Bm).createOzoneVolumeIfNeeded(null);
-
- Mockito.doNothing().when(mockBm).createBucket(null);
Mockito.doNothing().when(mockBm).createBucket(null);
Mockito.doNothing().when(mockBm).deleteBucket(null, null);
Mockito.doReturn(null).when(mockBm).getBucketInfo(null, null);
@@ -182,6 +169,7 @@
HddsWhiteboxTestUtils.setInternalState(
ozoneManager, "bucketManager", mockBm);
+
doBucketOps();
MetricsRecordBuilder omMetrics = getMetrics("OMMetrics");
@@ -198,28 +186,8 @@
ozoneManager.createBucket(null);
ozoneManager.deleteBucket(null, null);
- //Taking already existing value, as the same metrics is used over all the
- // test cases.
- long numVolumesOps = MetricsAsserts.getLongCounter("NumVolumeOps",
- omMetrics);
- long numVolumes = MetricsAsserts.getLongCounter("NumVolumes",
- omMetrics);
- long numVolumeCreates = MetricsAsserts.getLongCounter("NumVolumeCreates",
- omMetrics);
-
- ozoneManager.createS3Bucket("random", "random");
- ozoneManager.createS3Bucket("random1", "random1");
- ozoneManager.createS3Bucket("random2", "random2");
- ozoneManager.deleteS3Bucket("random");
-
omMetrics = getMetrics("OMMetrics");
- assertCounter("NumBuckets", 4L, omMetrics);
-
- assertCounter("NumVolumeOps", numVolumesOps + 3, omMetrics);
- assertCounter("NumVolumeCreates", numVolumeCreates + 3, omMetrics);
- assertCounter("NumVolumes", numVolumes + 3, omMetrics);
-
-
+ assertCounter("NumBuckets", 2L, omMetrics);
// inject exception to test for Failure Metrics
Mockito.doThrow(exception).when(mockBm).createBucket(null);
@@ -233,11 +201,11 @@
doBucketOps();
omMetrics = getMetrics("OMMetrics");
- assertCounter("NumBucketOps", 18L, omMetrics);
- assertCounter("NumBucketCreates", 8L, omMetrics);
+ assertCounter("NumBucketOps", 14L, omMetrics);
+ assertCounter("NumBucketCreates", 5L, omMetrics);
assertCounter("NumBucketUpdates", 2L, omMetrics);
assertCounter("NumBucketInfos", 2L, omMetrics);
- assertCounter("NumBucketDeletes", 4L, omMetrics);
+ assertCounter("NumBucketDeletes", 3L, omMetrics);
assertCounter("NumBucketLists", 2L, omMetrics);
assertCounter("NumBucketCreateFails", 1L, omMetrics);
@@ -246,10 +214,10 @@
assertCounter("NumBucketDeleteFails", 1L, omMetrics);
assertCounter("NumBucketListFails", 1L, omMetrics);
- assertCounter("NumBuckets", 4L, omMetrics);
+ assertCounter("NumBuckets", 2L, omMetrics);
cluster.restartOzoneManager();
- assertCounter("NumBuckets", 4L, omMetrics);
+ assertCounter("NumBuckets", 2L, omMetrics);
}
@Test
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java
index 9a30ed0..2feccef 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java
@@ -90,9 +90,10 @@
import static org.apache.hadoop.ozone.OzoneConfigKeys
.OZONE_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY;
import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_CLIENT_FAILOVER_SLEEP_BASE_MILLIS_DEFAULT;
+ .OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys
.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.DIRECTORY_NOT_FOUND;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_ALREADY_EXISTS;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_A_FILE;
import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType.USER;
@@ -108,7 +109,6 @@
/**
* Test Ozone Manager operation in distributed handler scenario.
*/
-@Ignore
public class TestOzoneManagerHA {
private MiniOzoneHAClusterImpl cluster = null;
@@ -282,6 +282,7 @@
/**
* Test client request fails when 2 OMs are down.
*/
+ @Ignore("This test is failing randomly. It will be enabled after fixing it.")
@Test
public void testTwoOMNodesDown() throws Exception {
cluster.stopOzoneManager(1);
@@ -394,7 +395,7 @@
try {
testCreateFile(ozoneBucket, keyName, data, false, false);
} catch (OMException ex) {
- Assert.assertEquals(NOT_A_FILE, ex.getResult());
+ Assert.assertEquals(DIRECTORY_NOT_FOUND, ex.getResult());
}
// create directory, now this should pass.
@@ -501,6 +502,38 @@
Assert.assertTrue(leaderOMNodeId != newLeaderOMNodeId);
}
+ /**
+ * 1. Stop one of the OM
+ * 2. make a call to OM, this will make failover attempts to find new node.
+ * a) if LE finishes but leader not ready, it retries to same node
+ * b) if LE not done, it will failover to new node and check
+ * 3. Try failover to same OM explicitly.
+ * Now #3 should wait additional waitBetweenRetries time.
+ * LE: Leader Election.
+ */
+ @Test
+ public void testIncrementalWaitTimeWithSameNodeFailover() throws Exception {
+ long waitBetweenRetries = conf.getLong(
+ OzoneConfigKeys.OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_KEY,
+ OzoneConfigKeys.OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_DEFAULT);
+ OMFailoverProxyProvider omFailoverProxyProvider =
+ objectStore.getClientProxy().getOMProxyProvider();
+
+ // The OMFailoverProxyProvider will point to the current leader OM node.
+ String leaderOMNodeId = omFailoverProxyProvider.getCurrentProxyOMNodeId();
+
+ cluster.stopOzoneManager(leaderOMNodeId);
+ Thread.sleep(NODE_FAILURE_TIMEOUT * 2);
+ createKeyTest(true); // failover should happen to new node
+
+ long numTimesTriedToSameNode = omFailoverProxyProvider.getWaitTime()
+ / waitBetweenRetries;
+ omFailoverProxyProvider.performFailoverIfRequired(omFailoverProxyProvider.
+ getCurrentProxyOMNodeId());
+ Assert.assertEquals((numTimesTriedToSameNode + 1) * waitBetweenRetries,
+ omFailoverProxyProvider.getWaitTime());
+ }
+
private String initiateMultipartUpload(OzoneBucket ozoneBucket,
String keyName) throws Exception {
@@ -672,6 +705,7 @@
/**
* Test OMFailoverProxyProvider failover on connection exception to OM client.
*/
+ @Ignore("This test randomly failing. Let's enable once its fixed.")
@Test
public void testOMProxyProviderFailoverOnConnectionFailure()
throws Exception {
@@ -684,11 +718,11 @@
// On stopping the current OM Proxy, the next connection attempt should
// failover to a another OM proxy.
cluster.stopOzoneManager(firstProxyNodeId);
- Thread.sleep(OZONE_CLIENT_FAILOVER_SLEEP_BASE_MILLIS_DEFAULT * 4);
+ Thread.sleep(OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_DEFAULT * 4);
// Next request to the proxy provider should result in a failover
createVolumeTest(true);
- Thread.sleep(OZONE_CLIENT_FAILOVER_SLEEP_BASE_MILLIS_DEFAULT);
+ Thread.sleep(OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_DEFAULT);
// Get the new OM Proxy NodeId
String newProxyNodeId = omFailoverProxyProvider.getCurrentProxyOMNodeId();
@@ -753,19 +787,10 @@
// the RpcClient should give up.
fail("TestOMRetryProxy should fail when there are no OMs running");
} catch (ConnectException e) {
- // Each retry attempt tries IPC_CLIENT_CONNECT_MAX_RETRIES times.
- // So there should be at least
- // OZONE_CLIENT_FAILOVER_MAX_ATTEMPTS * IPC_CLIENT_CONNECT_MAX_RETRIES
- // "Retrying connect to server" messages.
- // Also, the first call will result in EOFException.
- // That will result in another IPC_CLIENT_CONNECT_MAX_RETRIES attempts.
- Assert.assertEquals(
- (OZONE_CLIENT_FAILOVER_MAX_ATTEMPTS + 1) *
- IPC_CLIENT_CONNECT_MAX_RETRIES,
- appender.countLinesWithMessage("Retrying connect to server:"));
-
Assert.assertEquals(1,
appender.countLinesWithMessage("Failed to connect to OMs:"));
+ Assert.assertEquals(OZONE_CLIENT_FAILOVER_MAX_ATTEMPTS,
+ appender.countLinesWithMessage("Trying to failover"));
Assert.assertEquals(1,
appender.countLinesWithMessage("Attempted " +
OZONE_CLIENT_FAILOVER_MAX_ATTEMPTS + " failovers."));
@@ -1297,6 +1322,7 @@
}
+ @Ignore("This test randomly failing. Let's enable once its fixed.")
@Test
public void testListVolumes() throws Exception {
String userName = UserGroupInformation.getCurrentUser().getUserName();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListVolumes.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListVolumes.java
new file mode 100644
index 0000000..54b7804
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListVolumes.java
@@ -0,0 +1,240 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneAcl;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
+import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
+import org.apache.hadoop.security.UserGroupInformation;
+
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_RATIS_PIPELINE_LIMIT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS_NATIVE;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_VOLUME_LISTALL_ALLOWED;
+import static org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType.OZONE;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+/**
+ * Test OzoneManager list volume operation under combinations of configs.
+ */
+public class TestOzoneManagerListVolumes {
+
+ @Rule
+ public Timeout timeout = new Timeout(120_000);
+
+ private UserGroupInformation adminUser =
+ UserGroupInformation.createUserForTesting("om", new String[]{"ozone"});
+ private UserGroupInformation user1 =
+ UserGroupInformation.createUserForTesting("user1", new String[]{"test"});
+ private UserGroupInformation user2 =
+ UserGroupInformation.createUserForTesting("user2", new String[]{"test"});
+
+ @Before
+ public void init() throws Exception {
+ // loginUser is the user running this test.
+ // Implication: loginUser is automatically added to the OM admin list.
+ UserGroupInformation.setLoginUser(adminUser);
+ }
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ */
+ private MiniOzoneCluster startCluster(boolean aclEnabled,
+ boolean volListAllAllowed) throws Exception {
+
+ OzoneConfiguration conf = new OzoneConfiguration();
+ String clusterId = UUID.randomUUID().toString();
+ String scmId = UUID.randomUUID().toString();
+ String omId = UUID.randomUUID().toString();
+ conf.setInt(OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS, 2);
+ conf.setInt(OZONE_SCM_RATIS_PIPELINE_LIMIT, 10);
+
+ // Use native impl here, default impl doesn't do actual checks
+ conf.set(OZONE_ACL_AUTHORIZER_CLASS, OZONE_ACL_AUTHORIZER_CLASS_NATIVE);
+ // Note: OM doesn't support live config reloading
+ conf.setBoolean(OZONE_ACL_ENABLED, aclEnabled);
+ conf.setBoolean(OZONE_OM_VOLUME_LISTALL_ALLOWED, volListAllAllowed);
+
+ MiniOzoneCluster cluster = MiniOzoneCluster.newBuilder(conf)
+ .setClusterId(clusterId).setScmId(scmId).setOmId(omId).build();
+ cluster.waitForClusterToBeReady();
+
+ // Create volumes with non-default owners and ACLs
+ OzoneClient client = cluster.getClient();
+ ObjectStore objectStore = client.getObjectStore();
+
+ /* r = READ, w = WRITE, c = CREATE, d = DELETE
+ l = LIST, a = ALL, n = NONE, x = READ_ACL, y = WRITE_ACL */
+ String aclUser1All = "user:user1:a";
+ String aclUser2All = "user:user2:a";
+ String aclWorldAll = "world::a";
+ createVolumeWithOwnerAndAcl(objectStore, "volume1", "user1", aclUser1All);
+ createVolumeWithOwnerAndAcl(objectStore, "volume2", "user2", aclUser2All);
+ createVolumeWithOwnerAndAcl(objectStore, "volume3", "user1", aclUser2All);
+ createVolumeWithOwnerAndAcl(objectStore, "volume4", "user2", aclUser1All);
+ createVolumeWithOwnerAndAcl(objectStore, "volume5", "user1", aclWorldAll);
+
+ return cluster;
+ }
+
+ private void stopCluster(MiniOzoneCluster cluster) {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ private void createVolumeWithOwnerAndAcl(ObjectStore objectStore,
+ String volumeName, String ownerName, String aclString)
+ throws IOException {
+ ClientProtocol proxy = objectStore.getClientProxy();
+ objectStore.createVolume(volumeName);
+ proxy.setVolumeOwner(volumeName, ownerName);
+ setVolumeAcl(objectStore, volumeName, aclString);
+ }
+
+ /**
+ * Helper function to set volume ACL.
+ */
+ private void setVolumeAcl(ObjectStore objectStore, String volumeName,
+ String aclString) throws IOException {
+ OzoneObj obj = OzoneObjInfo.Builder.newBuilder().setVolumeName(volumeName)
+ .setResType(OzoneObj.ResourceType.VOLUME).setStoreType(OZONE).build();
+ Assert.assertTrue(objectStore.setAcl(obj, OzoneAcl.parseAcls(aclString)));
+ }
+
+ /**
+ * Helper function to reduce code redundancy for test checks with each user
+ * under different config combination.
+ */
+ private void checkUser(MiniOzoneCluster cluster, UserGroupInformation user,
+ List<String> expectVol, boolean expectListAllSuccess) throws IOException {
+
+ UserGroupInformation.setLoginUser(user);
+ OzoneClient client = cluster.getClient();
+ ObjectStore objectStore = client.getObjectStore();
+
+ // `ozone sh volume list` shall return volumes with LIST permission of user.
+ Iterator<? extends OzoneVolume> it = objectStore.listVolumesByUser(
+ user.getUserName(), "", "");
+ Set<String> accessibleVolumes = new HashSet<>();
+ while (it.hasNext()) {
+ OzoneVolume vol = it.next();
+ String volumeName = vol.getName();
+ accessibleVolumes.add(volumeName);
+ }
+ Assert.assertEquals(new HashSet<>(expectVol), accessibleVolumes);
+
+ // `ozone sh volume list --all` returns all volumes,
+ // or throws exception (for non-admin if acl enabled & listall disallowed).
+ if (expectListAllSuccess) {
+ it = objectStore.listVolumes("volume");
+ int count = 0;
+ while (it.hasNext()) {
+ it.next();
+ count++;
+ }
+ Assert.assertEquals(5, count);
+ } else {
+ try {
+ objectStore.listVolumes("volume");
+ Assert.fail("listAllVolumes should fail for " + user.getUserName());
+ } catch (RuntimeException ex) {
+ // Current listAllVolumes throws RuntimeException
+ if (ex.getCause() instanceof OMException) {
+ // Expect PERMISSION_DENIED
+ if (((OMException) ex.getCause()).getResult() !=
+ OMException.ResultCodes.PERMISSION_DENIED) {
+ throw ex;
+ }
+ } else {
+ throw ex;
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testAclEnabledListAllAllowed() throws Exception {
+ // ozone.acl.enabled = true, ozone.om.volume.listall.allowed = true
+ MiniOzoneCluster cluster = startCluster(true, true);
+ checkUser(cluster, user1, Arrays.asList("volume1", "volume4", "volume5"),
+ true);
+ checkUser(cluster, user2, Arrays.asList("volume2", "volume3", "volume5"),
+ true);
+ checkUser(cluster, adminUser, Arrays.asList("volume1", "volume2", "volume3",
+ "volume4", "volume5"), true);
+ stopCluster(cluster);
+ }
+
+ @Test
+ public void testAclEnabledListAllDisallowed() throws Exception {
+ // ozone.acl.enabled = true, ozone.om.volume.listall.allowed = false
+ MiniOzoneCluster cluster = startCluster(true, false);
+ checkUser(cluster, user1, Arrays.asList("volume1", "volume4", "volume5"),
+ false);
+ checkUser(cluster, user2, Arrays.asList("volume2", "volume3", "volume5"),
+ false);
+ checkUser(cluster, adminUser, Arrays.asList("volume1", "volume2", "volume3",
+ "volume4", "volume5"), true);
+ stopCluster(cluster);
+ }
+
+ @Test
+ public void testAclDisabledListAllAllowed() throws Exception {
+ // ozone.acl.enabled = false, ozone.om.volume.listall.allowed = true
+ MiniOzoneCluster cluster = startCluster(false, true);
+ checkUser(cluster, user1, Arrays.asList("volume1", "volume3", "volume5"),
+ true);
+ checkUser(cluster, user2, Arrays.asList("volume2", "volume4"),
+ true);
+ stopCluster(cluster);
+ }
+
+ @Test
+ public void testAclDisabledListAllDisallowed() throws Exception {
+ // ozone.acl.enabled = false, ozone.om.volume.listall.allowed = false
+ MiniOzoneCluster cluster = startCluster(false, false);
+ checkUser(cluster, user1, Arrays.asList("volume1", "volume3", "volume5"),
+ true);
+ checkUser(cluster, user2, Arrays.asList("volume2", "volume4"),
+ true); // listall will succeed since acl is disabled
+ stopCluster(cluster);
+ }
+}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmSafeMode.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmSafeMode.java
index 87c8f23..6c858ab 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmSafeMode.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmSafeMode.java
@@ -317,7 +317,7 @@
final List<ContainerInfo> containers = scm.getContainerManager()
.getContainers();
scm.getEventQueue().fireEvent(SCMEvents.SAFE_MODE_STATUS,
- new SCMSafeModeManager.SafeModeStatus(true));
+ new SCMSafeModeManager.SafeModeStatus(true, true));
GenericTestUtils.waitFor(() -> {
return clientProtocolServer.getSafeModeStatus();
}, 50, 1000 * 30);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/parser/TestOMRatisLogParser.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/parser/TestOMRatisLogParser.java
index d67bb5c..88cfc62 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/parser/TestOMRatisLogParser.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/parser/TestOMRatisLogParser.java
@@ -24,6 +24,7 @@
import org.apache.hadoop.ozone.om.helpers.OMRatisHelper;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.segmentparser.OMRatisLogParser;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -83,7 +84,7 @@
}
@Test
- public void testRatisLogParsing() {
+ public void testRatisLogParsing() throws Exception {
OzoneConfiguration ozoneConfiguration =
cluster.getOMLeader().getConfiguration();
@@ -94,22 +95,24 @@
Assert.assertTrue(omMetaDir.isDirectory());
String[] ratisDirs = omMetaDir.list();
- File groupDir = null;
-
+ Assert.assertNotNull(ratisDirs);
Assert.assertEquals(2, ratisDirs.length);
+ File groupDir = null;
for (int i=0; i< ratisDirs.length; i++) {
if (ratisDirs[i].equals("snapshot")) {
continue;
}
- groupDir = new File(omMetaDir, omMetaDir.list()[1]);
+ groupDir = new File(omMetaDir, ratisDirs[i]);
}
Assert.assertNotNull(groupDir);
+ Assert.assertFalse(groupDir.toString(),
+ groupDir.getName().contains("snapshot"));
Assert.assertTrue(groupDir.isDirectory());
File currentDir = new File(groupDir, "current");
File logFile = new File(currentDir, "log_inprogress_0");
- Assert.assertTrue(logFile.exists());
+ GenericTestUtils.waitFor(logFile::exists, 100, 15000);
Assert.assertTrue(logFile.isFile());
OMRatisLogParser omRatisLogParser = new OMRatisLogParser();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestS3Shell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestS3Shell.java
deleted file mode 100644
index 8b7fb1f..0000000
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestS3Shell.java
+++ /dev/null
@@ -1,292 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ozShell;
-
-import com.google.common.base.Strings;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hdds.client.ReplicationFactor;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
-import org.apache.hadoop.ozone.client.rpc.RpcClient;
-import org.apache.hadoop.ozone.om.exceptions.OMException;
-import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
-import org.apache.hadoop.ozone.web.ozShell.s3.S3Shell;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import picocli.CommandLine;
-import picocli.CommandLine.ExecutionException;
-import picocli.CommandLine.IExceptionHandler2;
-import picocli.CommandLine.ParameterException;
-import picocli.CommandLine.ParseResult;
-import picocli.CommandLine.RunLast;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort;
-
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.Arrays;
-import java.util.List;
-
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
-import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.S3_BUCKET_NOT_FOUND;
-import static org.apache.hadoop.ozone.web.ozShell.s3.GetS3SecretHandler.OZONE_GETS3SECRET_ERROR;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-/**
- * This test class specified for testing Ozone s3Shell command.
- */
-public class TestS3Shell {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(TestS3Shell.class);
-
- /**
- * Set the timeout for every test.
- */
- @Rule
- public Timeout testTimeout = new Timeout(300000);
-
- private static String url;
- private static File baseDir;
- private static OzoneConfiguration conf = null;
- private static MiniOzoneCluster cluster = null;
- private static ClientProtocol client = null;
- private static S3Shell s3Shell = null;
-
- private final ByteArrayOutputStream out = new ByteArrayOutputStream();
- private final ByteArrayOutputStream err = new ByteArrayOutputStream();
- private static final PrintStream OLD_OUT = System.out;
- private static final PrintStream OLD_ERR = System.err;
-
- /**
- * Create a MiniOzoneCluster for testing with using distributed Ozone
- * handler type.
- *
- * @throws Exception
- */
- @BeforeClass
- public static void init() throws Exception {
- conf = new OzoneConfiguration();
-
- String path = GenericTestUtils.getTempPath(
- TestS3Shell.class.getSimpleName());
- baseDir = new File(path);
- baseDir.mkdirs();
-
- s3Shell = new S3Shell();
-
- cluster = MiniOzoneCluster.newBuilder(conf)
- .setNumDatanodes(3)
- .build();
- conf.setInt(OZONE_REPLICATION, ReplicationFactor.THREE.getValue());
- conf.setQuietMode(false);
- client = new RpcClient(conf, null);
- cluster.waitForClusterToBeReady();
- }
-
- /**
- * shutdown MiniOzoneCluster.
- */
- @AfterClass
- public static void shutdown() {
- if (cluster != null) {
- cluster.shutdown();
- }
-
- if (baseDir != null) {
- FileUtil.fullyDelete(baseDir, true);
- }
- }
-
- @Before
- public void setup() {
- System.setOut(new PrintStream(out));
- System.setErr(new PrintStream(err));
- url = "o3://" + getOmAddress();
- }
-
- @After
- public void reset() {
- // reset stream after each unit test
- out.reset();
- err.reset();
-
- // restore system streams
- System.setOut(OLD_OUT);
- System.setErr(OLD_ERR);
- }
-
- @Test
- public void testS3BucketMapping() throws IOException {
- String setOmAddress =
- "--set=" + OZONE_OM_ADDRESS_KEY + "=" + getOmAddress();
-
- String s3Bucket = "bucket1";
- String commandOutput;
- createS3Bucket(OzoneConsts.OZONE, s3Bucket);
-
- // WHEN
- String[] args =
- new String[] {setOmAddress, "path", s3Bucket};
- execute(s3Shell, args);
-
- // THEN
- commandOutput = out.toString();
- String volumeName = client.getOzoneVolumeName(s3Bucket);
- assertTrue(commandOutput.contains("Volume name for S3Bucket is : " +
- volumeName));
- assertTrue(commandOutput.contains(OzoneConsts.OZONE_URI_SCHEME + "://" +
- s3Bucket + "." + volumeName));
- out.reset();
-
- // Trying to get map for an unknown bucket
- args = new String[] {setOmAddress, "path", "unknownbucket"};
- executeWithError(s3Shell, args, S3_BUCKET_NOT_FOUND);
-
- // No bucket name
- args = new String[] {setOmAddress, "path"};
- executeWithError(s3Shell, args, "Missing required parameter");
-
- // Invalid bucket name
- args = new String[] {setOmAddress, "path", "/asd/multipleslash"};
- executeWithError(s3Shell, args, S3_BUCKET_NOT_FOUND);
- }
-
- @Test
- public void testS3SecretUnsecuredCluster() throws Exception {
- String setOmAddress =
- "--set=" + OZONE_OM_ADDRESS_KEY + "=" + getOmAddress();
-
- String output;
-
- String[] args = new String[] {setOmAddress, "getsecret"};
- execute(s3Shell, args);
- // Get the first line of output
- output = out.toString().split("\n")[0];
-
- assertTrue(output.equals(OZONE_GETS3SECRET_ERROR));
- }
-
- private void createS3Bucket(String userName, String s3Bucket) {
- try {
- client.createS3Bucket(OzoneConsts.OZONE, s3Bucket);
- } catch (IOException ex) {
- GenericTestUtils.assertExceptionContains("S3_BUCKET_ALREADY_EXISTS", ex);
- }
- }
-
- private void execute(S3Shell shell, String[] args) {
- LOG.info("Executing s3Shell command with args {}", Arrays.asList(args));
- CommandLine cmd = shell.getCmd();
-
- IExceptionHandler2<List<Object>> exceptionHandler =
- new IExceptionHandler2<List<Object>>() {
- @Override
- public List<Object> handleParseException(ParameterException ex,
- String[] args) {
- throw ex;
- }
-
- @Override
- public List<Object> handleExecutionException(ExecutionException ex,
- ParseResult parseRes) {
- throw ex;
- }
- };
- cmd.parseWithHandlers(new RunLast(),
- exceptionHandler, args);
- }
-
- /**
- * Execute command, assert exception message and returns true if error
- * was thrown.
- */
- private void executeWithError(S3Shell shell, String[] args,
- OMException.ResultCodes code) {
- try {
- execute(shell, args);
- fail("Exception is expected from command execution " + Arrays
- .asList(args));
- } catch (Exception ex) {
- Assert.assertEquals(OMException.class, ex.getCause().getClass());
- Assert.assertEquals(code, ((OMException) ex.getCause()).getResult());
- }
- }
-
- /**
- * Execute command, assert exception message and returns true if error
- * was thrown.
- */
- private void executeWithError(S3Shell shell, String[] args,
- String expectedError) {
- if (Strings.isNullOrEmpty(expectedError)) {
- execute(shell, args);
- } else {
- try {
- execute(shell, args);
- fail("Exception is expected from command execution " + Arrays
- .asList(args));
- } catch (Exception ex) {
- if (!Strings.isNullOrEmpty(expectedError)) {
- Throwable exceptionToCheck = ex;
- if (exceptionToCheck.getCause() != null) {
- exceptionToCheck = exceptionToCheck.getCause();
- }
- Assert.assertTrue(
- String.format(
- "Error of s3Shell code doesn't contain the " +
- "exception [%s] in [%s]",
- expectedError, exceptionToCheck.getMessage()),
- exceptionToCheck.getMessage().contains(expectedError));
- }
- }
- }
- }
-
- private String getOmAddress() {
- List<ServiceInfo> services;
- try {
- services = cluster.getOzoneManager().getServiceList();
- } catch (IOException e) {
- fail("Could not get service list from OM");
- return null;
- }
-
- return services.stream()
- .filter(a -> HddsProtos.NodeType.OM.equals(a.getNodeType()))
- .findFirst()
- .map(s -> s.getServiceAddress(ServicePort.Type.RPC))
- .orElseThrow(IllegalStateException::new);
- }
-}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconAsPassiveScm.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconAsPassiveScm.java
index eabf667..07fbefd 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconAsPassiveScm.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconAsPassiveScm.java
@@ -81,7 +81,7 @@
}
}
- @Test(timeout = 120000)
+ @Test
public void testDatanodeRegistrationAndReports() throws Exception {
ReconStorageContainerManagerFacade reconScm =
(ReconStorageContainerManagerFacade)
@@ -91,7 +91,7 @@
PipelineManager scmPipelineManager = scm.getPipelineManager();
LambdaTestUtils.await(60000, 5000,
- () -> (reconPipelineManager.getPipelines().size() == 4));
+ () -> (reconPipelineManager.getPipelines().size() >= 4));
// Verify if Recon has all the pipelines from SCM.
scmPipelineManager.getPipelines().forEach(p -> {
@@ -139,7 +139,7 @@
1000, 20000);
}
- @Test(timeout = 120000)
+ @Test
public void testReconRestart() throws Exception {
final OzoneStorageContainerManager reconScm =
cluster.getReconServer().getReconStorageContainerManager();
@@ -193,7 +193,7 @@
assertFalse(
reconPipelineManager.containsPipeline(pipelineToClose.get().getId()));
- LambdaTestUtils.await(60000, 5000,
+ LambdaTestUtils.await(90000, 5000,
() -> (newReconScm.getContainerManager()
.exists(ContainerID.valueof(containerID))));
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java
new file mode 100644
index 0000000..08eb960
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.recon;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.RATIS;
+import static org.apache.hadoop.ozone.container.ozoneimpl.TestOzoneContainer.runTestOzoneContainerViaDataNode;
+import static org.junit.Assert.assertEquals;
+
+import java.util.List;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.XceiverClientGrpc;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerManager;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.recon.scm.ReconContainerManager;
+import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.hadoop.ozone.recon.schema.tables.pojos.MissingContainers;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+/**
+ * Integration Tests for Recon's tasks.
+ */
+public class TestReconTasks {
+
+ private MiniOzoneCluster cluster = null;
+ private OzoneConfiguration conf;
+
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ @Before
+ public void init() throws Exception {
+ conf = new OzoneConfiguration();
+ conf.set(HDDS_CONTAINER_REPORT_INTERVAL, "5s");
+ conf.set(HDDS_PIPELINE_REPORT_INTERVAL, "5s");
+ conf.set("ozone.recon.task.missingcontainer.interval", "15s");
+ conf.set("ozone.scm.stale.node.interval", "10s");
+ conf.set("ozone.scm.dead.node.interval", "20s");
+ cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1)
+ .includeRecon(true).build();
+ cluster.waitForClusterToBeReady();
+ }
+
+ @After
+ public void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testMissingContainerDownNode() throws Exception {
+ ReconStorageContainerManagerFacade reconScm =
+ (ReconStorageContainerManagerFacade)
+ cluster.getReconServer().getReconStorageContainerManager();
+ StorageContainerManager scm = cluster.getStorageContainerManager();
+ PipelineManager reconPipelineManager = reconScm.getPipelineManager();
+ PipelineManager scmPipelineManager = scm.getPipelineManager();
+
+ // Make sure Recon's pipeline state is initialized.
+ LambdaTestUtils.await(60000, 5000,
+ () -> (reconPipelineManager.getPipelines().size() >= 1));
+
+ ContainerManager scmContainerManager = scm.getContainerManager();
+ ReconContainerManager reconContainerManager =
+ (ReconContainerManager) reconScm.getContainerManager();
+ ContainerInfo containerInfo =
+ scmContainerManager.allocateContainer(RATIS, ONE, "test");
+ long containerID = containerInfo.getContainerID();
+ Pipeline pipeline =
+ scmPipelineManager.getPipeline(containerInfo.getPipelineID());
+ XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf);
+ runTestOzoneContainerViaDataNode(containerID, client);
+
+ // Make sure Recon got the container report with new container.
+ assertEquals(scmContainerManager.getContainerIDs(),
+ reconContainerManager.getContainerIDs());
+
+ // Bring down the Datanode that had the container replica.
+ cluster.shutdownHddsDatanode(pipeline.getFirstNode());
+
+ LambdaTestUtils.await(120000, 10000, () -> {
+ List<MissingContainers> allMissingContainers =
+ reconContainerManager.getContainerSchemaManager()
+ .getAllMissingContainers();
+ return (allMissingContainers.size() == 1);
+ });
+
+ // Restart the Datanode to make sure we remove the missing container.
+ cluster.restartHddsDatanode(pipeline.getFirstNode(), true);
+ LambdaTestUtils.await(120000, 10000, () -> {
+ List<MissingContainers> allMissingContainers =
+ reconContainerManager.getContainerSchemaManager()
+ .getAllMissingContainers();
+ return (allMissingContainers.isEmpty());
+ });
+ }
+}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java
deleted file mode 100644
index b36c323..0000000
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.scm.node;
-
-import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.NodeReportProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.StorageReportProto;
-import org.apache.hadoop.hdds.scm.TestUtils;
-import org.apache.hadoop.hdds.scm.node.SCMNodeMetrics;
-import org.apache.hadoop.metrics2.MetricsRecordBuilder;
-import org.apache.hadoop.ozone.HddsDatanodeService;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
-import static org.apache.hadoop.test.MetricsAsserts.assertGauge;
-import static org.apache.hadoop.test.MetricsAsserts.getLongCounter;
-import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
-
-/**
- * Test cases to verify the metrics exposed by SCMNodeManager.
- */
-public class TestSCMNodeMetrics {
-
- private MiniOzoneCluster cluster;
-
- @Before
- public void setup() throws Exception {
- OzoneConfiguration conf = new OzoneConfiguration();
- conf.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false);
- cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1).build();
- cluster.waitForClusterToBeReady();
- }
-
- /**
- * Verifies heartbeat processing count.
- *
- * @throws InterruptedException
- */
- @Test
- public void testHBProcessing() throws InterruptedException {
- MetricsRecordBuilder metrics = getMetrics(
- SCMNodeMetrics.class.getSimpleName());
- long hbProcessed = getLongCounter("NumHBProcessed", metrics);
- cluster.getHddsDatanodes().get(0)
- .getDatanodeStateMachine().triggerHeartbeat();
- // Give some time so that SCM receives and processes the heartbeat.
- Thread.sleep(100L);
- assertCounter("NumHBProcessed", hbProcessed + 1,
- getMetrics(SCMNodeMetrics.class.getSimpleName()));
- }
-
- /**
- * Verifies heartbeat processing failure count.
- */
- @Test
- public void testHBProcessingFailure() {
- MetricsRecordBuilder metrics = getMetrics(
- SCMNodeMetrics.class.getSimpleName());
- long hbProcessedFailed = getLongCounter("NumHBProcessingFailed", metrics);
- cluster.getStorageContainerManager().getScmNodeManager()
- .processHeartbeat(MockDatanodeDetails.randomDatanodeDetails());
- assertCounter("NumHBProcessingFailed", hbProcessedFailed + 1,
- getMetrics(SCMNodeMetrics.class.getSimpleName()));
- }
-
- /**
- * Verifies node report processing count.
- *
- * @throws InterruptedException
- */
- @Test
- public void testNodeReportProcessing() throws InterruptedException {
- MetricsRecordBuilder metrics = getMetrics(
- SCMNodeMetrics.class.getSimpleName());
- long nrProcessed = getLongCounter("NumNodeReportProcessed", metrics);
- HddsDatanodeService datanode = cluster.getHddsDatanodes().get(0);
- StorageReportProto storageReport = TestUtils.createStorageReport(
- datanode.getDatanodeDetails().getUuid(), "/tmp", 100, 10, 90, null);
- NodeReportProto nodeReport = NodeReportProto.newBuilder()
- .addStorageReport(storageReport).build();
- datanode.getDatanodeStateMachine().getContext().addReport(nodeReport);
- cluster.getStorageContainerManager().getScmNodeManager()
- .processNodeReport(datanode.getDatanodeDetails(), nodeReport);
-
- assertCounter("NumNodeReportProcessed", nrProcessed + 1,
- getMetrics(SCMNodeMetrics.class.getSimpleName()));
- }
-
- /**
- * Verifies node report processing failure count.
- */
- @Test
- public void testNodeReportProcessingFailure() {
- MetricsRecordBuilder metrics = getMetrics(
- SCMNodeMetrics.class.getSimpleName());
- long nrProcessed = getLongCounter("NumNodeReportProcessingFailed",
- metrics);
- DatanodeDetails datanode = MockDatanodeDetails.randomDatanodeDetails();
- StorageReportProto storageReport = TestUtils.createStorageReport(
- datanode.getUuid(), "/tmp", 100, 10, 90, null);
- NodeReportProto nodeReport = NodeReportProto.newBuilder()
- .addStorageReport(storageReport).build();
-
- cluster.getStorageContainerManager().getScmNodeManager()
- .processNodeReport(datanode, nodeReport);
- assertCounter("NumNodeReportProcessingFailed", nrProcessed + 1,
- getMetrics(SCMNodeMetrics.class.getSimpleName()));
- }
-
- /**
- * Verify that datanode aggregated state and capacity metrics are reported.
- */
- @Test
- public void testNodeCountAndInfoMetricsReported() throws Exception {
- HddsDatanodeService datanode = cluster.getHddsDatanodes().get(0);
- StorageReportProto storageReport = TestUtils.createStorageReport(
- datanode.getDatanodeDetails().getUuid(), "/tmp", 100, 10, 90, null);
- NodeReportProto nodeReport = NodeReportProto.newBuilder()
- .addStorageReport(storageReport).build();
- datanode.getDatanodeStateMachine().getContext().addReport(nodeReport);
- cluster.getStorageContainerManager().getScmNodeManager()
- .processNodeReport(datanode.getDatanodeDetails(), nodeReport);
-
- assertGauge("HealthyNodes", 1,
- getMetrics(SCMNodeMetrics.class.getSimpleName()));
- assertGauge("StaleNodes", 0,
- getMetrics(SCMNodeMetrics.class.getSimpleName()));
- assertGauge("DeadNodes", 0,
- getMetrics(SCMNodeMetrics.class.getSimpleName()));
- assertGauge("DecommissioningNodes", 0,
- getMetrics(SCMNodeMetrics.class.getSimpleName()));
- assertGauge("DecommissionedNodes", 0,
- getMetrics(SCMNodeMetrics.class.getSimpleName()));
- assertGauge("DiskCapacity", 100L,
- getMetrics(SCMNodeMetrics.class.getSimpleName()));
- assertGauge("DiskUsed", 10L,
- getMetrics(SCMNodeMetrics.class.getSimpleName()));
- assertGauge("DiskRemaining", 90L,
- getMetrics(SCMNodeMetrics.class.getSimpleName()));
- assertGauge("SSDCapacity", 0L,
- getMetrics(SCMNodeMetrics.class.getSimpleName()));
- assertGauge("SSDUsed", 0L,
- getMetrics(SCMNodeMetrics.class.getSimpleName()));
- assertGauge("SSDRemaining", 0L,
- getMetrics(SCMNodeMetrics.class.getSimpleName()));
- }
-
- @After
- public void teardown() {
- cluster.shutdown();
- }
-}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/pipeline/TestSCMPipelineBytesWrittenMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/pipeline/TestSCMPipelineBytesWrittenMetrics.java
index 396cf44..dc9e5e2 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/pipeline/TestSCMPipelineBytesWrittenMetrics.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/pipeline/TestSCMPipelineBytesWrittenMetrics.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.ozone.scm.pipeline;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/pipeline/TestSCMPipelineMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/pipeline/TestSCMPipelineMetrics.java
index cf0dcf4..f86c081 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/pipeline/TestSCMPipelineMetrics.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/pipeline/TestSCMPipelineMetrics.java
@@ -32,7 +32,6 @@
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Test;
import java.io.IOException;
@@ -45,7 +44,6 @@
/**
* Test cases to verify the metrics exposed by SCMPipelineManager.
*/
-@Ignore
public class TestSCMPipelineMetrics {
private MiniOzoneCluster cluster;
@@ -83,16 +81,15 @@
Optional<Pipeline> pipeline = pipelineManager
.getPipelines().stream().findFirst();
Assert.assertTrue(pipeline.isPresent());
- pipeline.ifPresent(pipeline1 -> {
- try {
- cluster.getStorageContainerManager()
- .getClientProtocolServer().closePipeline(
- pipeline.get().getId().getProtobuf());
- } catch (IOException e) {
- e.printStackTrace();
- Assert.fail();
- }
- });
+ try {
+ cluster.getStorageContainerManager()
+ .getPipelineManager()
+ .finalizeAndDestroyPipeline(
+ pipeline.get(), false);
+ } catch (IOException e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
MetricsRecordBuilder metrics = getMetrics(
SCMPipelineMetrics.class.getSimpleName());
assertCounter("NumPipelineDestroyed", 1L, metrics);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneDatanodeShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDatanodeShell.java
similarity index 98%
rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneDatanodeShell.java
rename to hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDatanodeShell.java
index 740baec..4510136 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneDatanodeShell.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDatanodeShell.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.ozone.ozShell;
+package org.apache.hadoop.ozone.shell;
import static org.junit.Assert.fail;
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShellHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
similarity index 93%
rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShellHA.java
rename to hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
index 1c35d5b..5fe294e 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShellHA.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.ozone.ozShell;
+package org.apache.hadoop.ozone.shell;
import com.google.common.base.Strings;
import org.apache.hadoop.fs.FileUtil;
@@ -26,10 +26,6 @@
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OzoneManager;
-import org.apache.hadoop.ozone.web.ozShell.OzoneShell;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import org.apache.hadoop.ozone.web.ozShell.s3.S3Shell;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.AfterClass;
@@ -444,31 +440,4 @@
Assert.assertEquals(0, getNumOfBuckets("bucket"));
}
-
- @Test
- public void testS3PathCommand() throws Exception {
-
- String s3Bucket = "b12345";
- cluster.getRpcClient().getObjectStore().createS3Bucket(
- UserGroupInformation.getCurrentUser().getUserName(), s3Bucket);
-
- String[] args = new String[] {"path", s3Bucket,
- "--om-service-id="+omServiceId};
-
- S3Shell s3Shell = new S3Shell();
- execute(s3Shell, args);
-
-
- String volumeName =
- cluster.getRpcClient().getObjectStore().getOzoneVolumeName(s3Bucket);
-
- String ozoneFsUri = String.format("%s://%s.%s", OzoneConsts
- .OZONE_URI_SCHEME, s3Bucket, volumeName);
-
- Assert.assertTrue(out.toString().contains(volumeName));
- Assert.assertTrue(out.toString().contains(ozoneFsUri));
-
- out.reset();
- }
-
}
diff --git a/hadoop-ozone/native-client/README.md b/hadoop-ozone/native-client/README.md
new file mode 100644
index 0000000..04bb6fa
--- /dev/null
+++ b/hadoop-ozone/native-client/README.md
@@ -0,0 +1,96 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+#Overview
+
+libo3fs is a JNI based C API for Ozone File System. It provides with read and write functionality on OzoneFileSystem. It also uses some functions from HDFS(Hadoop Distributed File System) for which it uses libhdfs, which is a JNI based C API for Hadoop’s Distributed File System (HDFS). It provides C APIs to a subset of the HDFS APIs to manipulate HDFS files and the filesystem. libhdfs is part of the Hadoop distribution and comes pre-compiled in $HADOOP_HDFS_HOME/lib/native/libhdfs.so .
+
+#The APIs
+
+The libo3fs APIs are a subset of the Ozone FileSystem APIs. The header file for libo3fs describes each API in detail and is available in ${OZONE_HOME}/native-client/libo3fs/o3fs.h.
+
+#Requirements:
+
+1.Hadoop with compiled libhdfs.so
+2.Linux kernel > 2.6.9
+3.Compiled Ozone
+
+#Compilation
+
+ 1.Compilation of .c files
+ In libo3fs directory there is one file o3fs.c.
+
+ Execute the following command to compile it:
+
+ gcc -fPIC -pthread -I {OZONE_HOME}/native-client/libo3fs -I {HADOOP_HOME}/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include -g -c o3fs.c
+ In the libo3fs-examples directory there are two .c files: libo3fs_read.c and libo3fs_write.c.
+
+ Execute the following command to compile libo3fs_read.c and libo3fs_write.c:
+
+ gcc -fPIC -pthread -I {OZONE_HOME}/native-client/libo3fs -I {HADOOP_HOME}/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include -g -c libo3fs_read.c
+ gcc -fPIC -pthread -I {OZONE_HOME}/native-client/libo3fs -I {HADOOP_HOME}/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include -g -c libo3fs_write.c
+
+ 2. Generation of libo3fs.so
+ Execute the following command to generate a .so:
+
+ gcc -shared o3fs.o hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/mybuild/hdfs.o -o libo3fs.so.
+
+ 3. Generation of binary(executable)
+ Two binaries have to be generated namely o3fs_read and o3fs_write.
+ Execute the following command to generate o3fs_red:
+
+ gcc -L {HADOOP_HOME}/hadoop-hdfs-project/hadoop-hdfs-native-client/target/native/target/usr/local/lib -o o3fs_read libo3fs_read.o -lhdfs -pthread -L/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.242.b08-0.el7_7.x86_64/jre/lib/amd64/server -ljvm -L {OZONE_HOME}/native-client/libo3fs -lo3fs
+
+ Execute the following command to execute o3fs_write:
+
+ gcc -L {HADOOP_HOME}/hadoop-hdfs-project/hadoop-hdfs-native-client/target/native/target/usr/local/lib -o o3fs_write libo3fs_write.o -lhdfs -pthread -L/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.242.b08- 0.el7_7.x86_64/jre/lib/amd64/server -ljvm -L {OZONE_HOME}/native-client/libo3fs -lo3fs
+
+#Deploying
+
+In root shell execute the following:
+
+ ./o3fs_write filename file_size buffer_size host_name port_number bucket_name volume_name
+
+For example
+
+ ./o3fs_write file1 100 100 127.0.0.1 9862 bucket4 vol4 , where file1 is name of the file, 100 is file size and buffer size, "127.0.0.1 is host", 9862 is the port number, "bucket4" is bucket name and "vol4" is the volume name.
+
+#Common Problems:
+
+CLASSPATH is not set. CLASSPATH can be set using the following command:
+
+ export CLASSPATH=$({OZONE_HOME}/hadoop-ozone/dist/target/ozone-0.5.0-SNAPSHOT/bin/ozone classpath hadoop-ozone-filesystem --glob)
+
+ export CLASSPATH=$CLASSPATH:{OZONE_HOME}/hadoop-ozone/dist/target/ozone-0.5.0-SNAPSHOT/share/ozone/lib/hadoop-ozone-filesystem-0.5.0-SNAPSHOT.jar
+
+LD_LIBRARY_PATH is not set. LD_LIBRARY_PATH can be set using the following command:
+
+ export LD_LIBRARY_PATH={HADOOP_HOME}/hadoop-hdfs-project/hadoop-hdfs-native-client/target/native/target/usr/local/lib:
+ /usr/lib/jvm/java-1.8.0-openjdk-1.8.0.242.b08-0.el7_7.x86_64/jre/lib/amd64/server:
+ {OZONE_HOME}/native-client/libo3fs
+
+Ozone-site.xml is not configured. Save the minimal snippet to hadoop-ozone/dist/target/ozone-*/etc/hadoop/ozone-site.xml in the compiled distribution.
+
+ <configuration>
+ <properties>
+ <property><name>ozone.scm.datanode.id.dir</name><value>/tmp/ozone/data</value></property>
+ <property><name>ozone.replication</name><value>1</value></property>
+ <property><name>ozone.metadata.dirs</name><value>/tmp/ozone/data/metadata</value></property>
+ <property><name>ozone.scm.names</name><value>localhost</value></property>
+ <property><name>ozone.om.address</name><value>localhost</value></property>
+ </properties>
+ </configuration>
diff --git a/hadoop-ozone/native-client/libo3fs-examples/libo3fs_read.c b/hadoop-ozone/native-client/libo3fs-examples/libo3fs_read.c
new file mode 100644
index 0000000..5cf727a
--- /dev/null
+++ b/hadoop-ozone/native-client/libo3fs-examples/libo3fs_read.c
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "o3fs.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include<string.h>
+
+int main(int argc, char **argv) {
+ o3fsFS fs;
+ const char *rfile = argv[1];
+ tSize bufferSize = strtoul(argv[3], NULL, 10);
+ const char *host = argv[4];
+ tPort port = atoi(argv[5]);
+ const char *bucket = argv[6];
+ const char *volume = argv[7];
+ o3fsFile readFile;
+ char* buffer;
+ tSize curSize;
+ char message[110] = "Usage: o3fs_read <filename> <filesize> <buffersize>";
+ strcat(message, " <host-name> <port> <bucket-name> <volume-name>\n");
+ if (argc != 8) {
+ fprintf(stderr, message);
+ exit(-1);
+ }
+ fs = o3fsConnect(host, port, bucket, volume);
+ if (!fs) {
+ fprintf(stderr, "Oops! Failed to connect to o3fs!\n");
+ exit(-1);
+ }
+ readFile = o3fsOpenFile(fs, rfile, O_RDONLY, bufferSize, 0, 0);
+ if (!readFile) {
+ fprintf(stderr, "Failed to open %s for writing!\n", rfile);
+ exit(-2);
+ }
+ // data to be written to the file
+ buffer = malloc(sizeof(char) * bufferSize);
+ if(buffer == NULL) {
+ return -2;
+ }
+ // read from the file
+ curSize = bufferSize;
+ for (; curSize == bufferSize;) {
+ curSize = o3fsRead(fs, readFile, (void*)buffer, curSize);
+ }
+ free(buffer);
+ o3fsCloseFile(fs, readFile);
+ o3fsDisconnect(fs);
+ return 0;
+}
\ No newline at end of file
diff --git a/hadoop-ozone/native-client/libo3fs-examples/libo3fs_write.c b/hadoop-ozone/native-client/libo3fs-examples/libo3fs_write.c
new file mode 100644
index 0000000..30c8683
--- /dev/null
+++ b/hadoop-ozone/native-client/libo3fs-examples/libo3fs_write.c
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "o3fs.h"
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/types.h>
+#include<string.h>
+
+int main(int argc, char **argv) {
+ o3fsFS fs;
+ const char *writeFileName = argv[1];
+ off_t fileTotalSize = strtoul(argv[2], NULL, 10);
+ long long tmpBufferSize = strtoul(argv[3], NULL, 10);
+ const char *host = argv[4];
+ tPort port = atoi(argv[5]);
+ const char *bucket = argv[6];
+ const char *volume = argv[7];
+ tSize bufferSize;
+ o3fsFile writeFile;
+ char* buffer;
+ int i;
+ off_t nrRemaining;
+ tSize curSize;
+ tSize written;
+ char message[110] = "Usage: o3fs_write <filename> <filesize> <buffersize>";
+ strcat(message, " <host-name> <port> <bucket-name> <volume-name>\n");
+ if (argc != 8) {
+ fprintf(stderr, message);
+ exit(-1);
+ }
+ fs = o3fsConnect(host, port, bucket, volume);
+ if (!fs) {
+ fprintf(stderr, "Oops! Failed to connect to o3fs!\n");
+ exit(-1);
+ }
+ if(fileTotalSize == ULONG_MAX && errno == ERANGE) {
+ fprintf(stderr, "invalid file size %s - must be <= %lu\n",
+ argv[2], ULONG_MAX);
+ exit(-3);
+ }
+ if(tmpBufferSize > INT_MAX) {
+ fprintf(stderr,
+ "invalid buffer size libhdfs API write chunks must be <= %d\n", INT_MAX);
+ exit(-3);
+ }
+ bufferSize = (tSize)tmpBufferSize;
+ writeFile = o3fsOpenFile(fs, writeFileName, O_WRONLY, bufferSize, 0, 0);
+ if (!writeFile) {
+ fprintf(stderr, "Failed to open %s for writing!\n", writeFileName);
+ exit(-2);
+ }
+ buffer = malloc(sizeof(char) * bufferSize);
+ if(buffer == NULL) {
+ fprintf(stderr, "Could not allocate buffer of size %d\n", bufferSize);
+ return -2;
+ }
+ for (i=0; i < bufferSize; ++i) {
+ buffer[i] = 'a' + (i%26);
+ }
+ for (nrRemaining = fileTotalSize; nrRemaining > 0;
+ nrRemaining -= bufferSize ) {
+ curSize = ( bufferSize < nrRemaining ) ? bufferSize : (tSize)nrRemaining;
+ if ((written = o3fsWrite(fs, writeFile, (void*)buffer,
+ curSize)) != curSize) {
+ fprintf(stderr, "ERROR: o3fsWrite returned an error on write: %d\n",
+ written);
+ exit(-3);
+ }
+ }
+ free(buffer);
+ o3fsCloseFile(fs, writeFile);
+ o3fsDisconnect(fs);
+ return 0;
+}
\ No newline at end of file
diff --git a/hadoop-ozone/native-client/libo3fs/o3fs.c b/hadoop-ozone/native-client/libo3fs/o3fs.c
new file mode 100644
index 0000000..421b7a4
--- /dev/null
+++ b/hadoop-ozone/native-client/libo3fs/o3fs.c
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "o3fs.h"
+#include "hdfs/hdfs.h"
+#include <fcntl.h>
+#include <inttypes.h>
+#include <stdio.h>
+#include <string.h>
+#define O3FS "o3fs://"
+
+o3fsFS o3fsConnect(const char *host, tPort port,
+ const char *bucket, const char *vol)
+{
+ struct hdfsBuilder *bld = hdfsNewBuilder();
+ int len = 0;
+ if (!bld){
+ return NULL;
+ }
+ len = strlen(host) + strlen(bucket) + strlen(vol) + strlen(O3FS);
+ char string[len + 2];
+ snprintf(string, len + 3, "%s%s.%s.%s", O3FS, bucket, vol, host);
+ // After snprintf command,
+ // string = o3fs://bucket4.vol4.127.0.0.1 (URI without port)
+ //port will be added to URI in hdfsBuilerConnect() function below.
+ //finally URI: o3fs://bucket4.vol4.127.0.0.1:9862
+ hdfsBuilderSetNameNode(bld, string);
+ hdfsBuilderSetNameNodePort(bld, port);
+ return (o3fsFS)hdfsBuilderConnect(bld);
+}
+o3fsFile o3fsOpenFile(o3fsFS fs, const char *path, int flags, int bufferSize,
+ short replication, tSize blockSize){
+ return (o3fsFile)hdfsOpenFile((hdfsFS)fs, path, flags, bufferSize,
+ replication, blockSize);
+}
+
+tSize o3fsRead(o3fsFS fs, o3fsFile f, void* buffer, tSize length){
+ return hdfsRead((hdfsFS)fs, (hdfsFile)f, buffer, length);
+}
+
+int o3fsCloseFile(o3fsFS fs, o3fsFile file){
+ return hdfsCloseFile((hdfsFS)fs, (hdfsFile)file);
+}
+
+int o3fsDisconnect(o3fsFS fs){
+ return hdfsDisconnect((hdfsFS)fs);
+}
+
+tSize o3fsWrite(o3fsFS fs, o3fsFile f, const void* buffer, tSize length){
+ return hdfsWrite((hdfsFS)fs, (hdfsFile)f, buffer, length);
+}
diff --git a/hadoop-ozone/native-client/libo3fs/o3fs.h b/hadoop-ozone/native-client/libo3fs/o3fs.h
new file mode 100644
index 0000000..d908900
--- /dev/null
+++ b/hadoop-ozone/native-client/libo3fs/o3fs.h
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef O3FS_DOT_H
+#define O3FS_DOT_H
+
+#include "hdfs/hdfs.h"
+
+struct hdfs_internal;
+typedef struct hdfs_internal* o3fsFS;
+
+struct hdfsFile_internal;
+typedef struct hdfsFile_internal* o3fsFile;
+
+o3fsFS o3fsConnect(const char* nn, tPort port, const char* bucket,
+ const char* volume);
+
+o3fsFile o3fsOpenFile(o3fsFS fs, const char *path, int flags, int bufferSize,
+ short replication, tSize blockSize);
+
+tSize o3fsRead(o3fsFS fs, o3fsFile f, void* buffer, tSize length);
+
+int o3fsCloseFile(o3fsFS fs, o3fsFile file);
+
+int o3fsDisconnect(o3fsFS fs);
+
+tSize o3fsWrite(o3fsFS fs, o3fsFile f, const void* buffer, tSize length);
+
+#endif
\ No newline at end of file
diff --git a/hadoop-ozone/ozone-manager/pom.xml b/hadoop-ozone/ozone-manager/pom.xml
index 5c5c3b4..9f02e79 100644
--- a/hadoop-ozone/ozone-manager/pom.xml
+++ b/hadoop-ozone/ozone-manager/pom.xml
@@ -47,8 +47,17 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-test-utils</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-server-framework</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-hadoop-dependency-server</artifactId>
+ </dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
@@ -67,11 +76,9 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
+ <artifactId>hadoop-hdds-hadoop-dependency-test</artifactId>
<scope>test</scope>
- <type>test-jar</type>
</dependency>
-
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-server-scm</artifactId>
@@ -88,7 +95,6 @@
<dependency>
<groupId>org.jmockit</groupId>
<artifactId>jmockit</artifactId>
- <version>1.24</version>
<scope>test</scope>
</dependency>
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyDeletingService.java
index 27ad7d8..1f48334 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyDeletingService.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyDeletingService.java
@@ -26,7 +26,7 @@
import com.google.protobuf.ServiceException;
import org.apache.commons.lang3.tuple.Pair;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
import org.apache.hadoop.ozone.common.BlockGroup;
import org.apache.hadoop.ozone.common.DeleteBlockGroupResult;
@@ -80,7 +80,7 @@
KeyDeletingService(OzoneManager ozoneManager,
ScmBlockLocationProtocol scmClient,
KeyManager manager, long serviceInterval,
- long serviceTimeout, Configuration conf) {
+ long serviceTimeout, ConfigurationSource conf) {
super("KeyDeletingService", serviceInterval, TimeUnit.MILLISECONDS,
KEY_DELETING_CORE_POOL_SIZE, serviceTimeout);
this.ozoneManager = ozoneManager;
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index d03fbd9..62f0015 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -26,6 +26,7 @@
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -37,8 +38,6 @@
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
@@ -106,6 +105,8 @@
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED_DEFAULT;
@@ -687,36 +688,45 @@
*/
@VisibleForTesting
protected void refreshPipeline(OmKeyInfo value) throws IOException {
- if (value != null &&
- CollectionUtils.isNotEmpty(value.getKeyLocationVersions())) {
- Map<Long, ContainerWithPipeline> containerWithPipelineMap =
- new HashMap<>();
- for (OmKeyLocationInfoGroup key : value.getKeyLocationVersions()) {
- for (OmKeyLocationInfo k : key.getLocationList()) {
- // TODO: fix Some tests that may not initialize container client
- // The production should always have containerClient initialized.
- if (scmClient.getContainerClient() != null) {
- try {
- if (!containerWithPipelineMap.containsKey(k.getContainerID())) {
- ContainerWithPipeline containerWithPipeline = scmClient
- .getContainerClient()
- .getContainerWithPipeline(k.getContainerID());
- containerWithPipelineMap.put(k.getContainerID(),
- containerWithPipeline);
- }
- } catch (IOException ioEx) {
- LOG.debug("Get containerPipeline failed for volume:{} bucket:{} "
- + "key:{}", value.getVolumeName(), value.getBucketName(),
- value.getKeyName(), ioEx);
- throw new OMException(ioEx.getMessage(),
- SCM_GET_PIPELINE_EXCEPTION);
- }
- ContainerWithPipeline cp =
- containerWithPipelineMap.get(k.getContainerID());
- if (!cp.getPipeline().equals(k.getPipeline())) {
- k.setPipeline(cp.getPipeline());
- }
- }
+ final List<OmKeyLocationInfoGroup> locationInfoGroups = value == null ?
+ null : value.getKeyLocationVersions();
+
+ // TODO: fix Some tests that may not initialize container client
+ // The production should always have containerClient initialized.
+ if (scmClient.getContainerClient() == null ||
+ CollectionUtils.isEmpty(locationInfoGroups)) {
+ return;
+ }
+
+ Set<Long> containerIDs = new HashSet<>();
+ for (OmKeyLocationInfoGroup key : locationInfoGroups) {
+ for (OmKeyLocationInfo k : key.getLocationList()) {
+ containerIDs.add(k.getContainerID());
+ }
+ }
+
+ Map<Long, ContainerWithPipeline> containerWithPipelineMap = new HashMap<>();
+
+ try {
+ List<ContainerWithPipeline> cpList = scmClient.getContainerClient().
+ getContainerWithPipelineBatch(new ArrayList<>(containerIDs));
+ for (ContainerWithPipeline cp : cpList) {
+ containerWithPipelineMap.put(
+ cp.getContainerInfo().getContainerID(), cp);
+ }
+ } catch (IOException ioEx) {
+ LOG.debug("Get containerPipeline failed for volume:{} bucket:{} " +
+ "key:{}", value.getVolumeName(), value.getBucketName(),
+ value.getKeyName(), ioEx);
+ throw new OMException(ioEx.getMessage(), SCM_GET_PIPELINE_EXCEPTION);
+ }
+
+ for (OmKeyLocationInfoGroup key : locationInfoGroups) {
+ for (OmKeyLocationInfo k : key.getLocationList()) {
+ ContainerWithPipeline cp =
+ containerWithPipelineMap.get(k.getContainerID());
+ if (!cp.getPipeline().equals(k.getPipeline())) {
+ k.setPipeline(cp.getPipeline());
}
}
}
@@ -1604,8 +1614,15 @@
OzoneFileStatus fileStatus = getFileStatus(args);
keyInfo = fileStatus.getKeyInfo();
} catch (IOException e) {
- throw new OMException("Key not found, checkAccess failed. Key:" +
- objectKey, KEY_NOT_FOUND);
+ // OzoneFS will check whether the key exists when write a new key.
+ // For Acl Type "READ", when the key is not exist return true.
+ // To Avoid KEY_NOT_FOUND Exception.
+ if (context.getAclRights() == IAccessAuthorizer.ACLType.READ) {
+ return true;
+ } else {
+ throw new OMException("Key not found, checkAccess failed. Key:" +
+ objectKey, KEY_NOT_FOUND);
+ }
}
}
@@ -1684,7 +1701,7 @@
// Check if this is the root of the filesystem.
if (keyName.length() == 0) {
validateBucket(volumeName, bucketName);
- return new OzoneFileStatus(OZONE_URI_DELIMITER);
+ return new OzoneFileStatus();
}
// Check if the key is a file.
@@ -1748,7 +1765,7 @@
Path keyPath = Paths.get(keyName);
OzoneFileStatus status =
verifyNoFilesInPath(volumeName, bucketName, keyPath, false);
- if (status != null && OzoneFSUtils.pathToKey(status.getPath())
+ if (status != null && status.getTrimmedName()
.equals(keyName)) {
// if directory already exists
return;
@@ -2020,8 +2037,13 @@
// if entry is a directory
if (!deletedKeySet.contains(entryInDb)) {
if (!entryKeyName.equals(immediateChild)) {
+ OmKeyInfo fakeDirEntry = new OmKeyInfo.Builder()
+ .setVolumeName(omKeyInfo.getVolumeName())
+ .setBucketName(omKeyInfo.getBucketName())
+ .setKeyName(immediateChild)
+ .build();
cacheKeyMap.put(entryInDb,
- new OzoneFileStatus(immediateChild));
+ new OzoneFileStatus(fakeDirEntry, scmBlockSize, true));
} else {
// If entryKeyName matches dir name, we have the info
cacheKeyMap.put(entryInDb,
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
index 43bd0d7..ac5d869 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
@@ -18,29 +18,39 @@
package org.apache.hadoop.ozone.om;
-import static org.apache.hadoop.ozone.OzoneConsts.OM_RATIS_SNAPSHOT_BEFORE_DB_CHECKPOINT;
-import static org.apache.hadoop.ozone.OzoneConsts.OM_RATIS_SNAPSHOT_INDEX;
-import static org.apache.hadoop.ozone.OzoneConsts.OM_RATIS_SNAPSHOT_TERM;
-import static org.apache.hadoop.ozone.OzoneConsts.
- OZONE_DB_CHECKPOINT_REQUEST_FLUSH;
-
-import java.io.IOException;
-import java.nio.file.Path;
-import java.time.Duration;
-import java.time.Instant;
-
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
-import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdfs.util.DataTransferThrottler;
-import org.apache.hadoop.ozone.OmUtils;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.hdds.utils.db.DBStore;
import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdfs.util.DataTransferThrottler;
+import org.apache.hadoop.ozone.OzoneConsts;
+
+import org.apache.commons.compress.archivers.ArchiveEntry;
+import org.apache.commons.compress.archivers.ArchiveOutputStream;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.compress.compressors.CompressorException;
+import org.apache.commons.compress.compressors.CompressorOutputStream;
+import org.apache.commons.compress.compressors.CompressorStreamFactory;
+import org.apache.commons.compress.utils.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import static org.apache.hadoop.ozone.OzoneConsts.OM_RATIS_SNAPSHOT_BEFORE_DB_CHECKPOINT;
+import static org.apache.hadoop.ozone.OzoneConsts.OM_RATIS_SNAPSHOT_INDEX;
+import static org.apache.hadoop.ozone.OzoneConsts.OM_RATIS_SNAPSHOT_TERM;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_FLUSH;
import org.apache.ratis.server.protocol.TermIndex;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -154,7 +164,7 @@
file.toString() + ".tgz\"");
Instant start = Instant.now();
- OmUtils.writeOmDBCheckpointToStream(checkpoint,
+ writeOmDBCheckpointToStream(checkpoint,
response.getOutputStream());
Instant end = Instant.now();
@@ -180,4 +190,53 @@
}
}
+ /**
+ * Write OM DB Checkpoint to an output stream as a compressed file (tgz).
+ *
+ * @param checkpoint checkpoint file
+ * @param destination desination output stream.
+ * @throws IOException
+ */
+ public static void writeOmDBCheckpointToStream(DBCheckpoint checkpoint,
+ OutputStream destination)
+ throws IOException {
+
+ try (CompressorOutputStream gzippedOut = new CompressorStreamFactory()
+ .createCompressorOutputStream(CompressorStreamFactory.GZIP,
+ destination)) {
+
+ try (ArchiveOutputStream archiveOutputStream =
+ new TarArchiveOutputStream(gzippedOut)) {
+
+ Path checkpointPath = checkpoint.getCheckpointLocation();
+ try (Stream<Path> files = Files.list(checkpointPath)) {
+ for (Path path : files.collect(Collectors.toList())) {
+ if (path != null) {
+ Path fileName = path.getFileName();
+ if (fileName != null) {
+ includeFile(path.toFile(), fileName.toString(),
+ archiveOutputStream);
+ }
+ }
+ }
+ }
+ }
+ } catch (CompressorException e) {
+ throw new IOException(
+ "Can't compress the checkpoint: " +
+ checkpoint.getCheckpointLocation(), e);
+ }
+ }
+
+ private static void includeFile(File file, String entryName,
+ ArchiveOutputStream archiveOutputStream)
+ throws IOException {
+ ArchiveEntry archiveEntry =
+ archiveOutputStream.createArchiveEntry(file, entryName);
+ archiveOutputStream.putArchiveEntry(archiveEntry);
+ try (FileInputStream fis = new FileInputStream(file)) {
+ IOUtils.copy(fis, archiveOutputStream);
+ }
+ archiveOutputStream.closeArchiveEntry();
+ }
}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
similarity index 95%
rename from hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
rename to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
index 9a9cf80..e5432da 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
@@ -203,6 +203,18 @@
String startKeyName, String keyPrefix, int maxKeys) throws IOException;
/**
+ * Recover trash allows the user to recover the keys
+ * that were marked as deleted, but not actually deleted by Ozone Manager.
+ * @param volumeName - The volume name.
+ * @param bucketName - The bucket name.
+ * @param keyName - The key user want to recover.
+ * @param destinationBucket - The bucket user want to recover to.
+ * @return The result of recovering operation is success or not.
+ */
+ boolean recoverTrash(String volumeName, String bucketName,
+ String keyName, String destinationBucket) throws IOException;
+
+ /**
* Returns a list of volumes owned by a given user; if user is null, returns
* all volumes.
*
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStorage.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStorage.java
index 12a8017..c527af9 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStorage.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStorage.java
@@ -22,7 +22,7 @@
import java.util.Properties;
import java.util.UUID;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
import org.apache.hadoop.hdds.server.ServerUtils;
@@ -114,7 +114,7 @@
* @param conf - Config
* @return File path, after creating all the required Directories.
*/
- public static File getOmDbDir(Configuration conf) {
+ public static File getOmDbDir(ConfigurationSource conf) {
return ServerUtils.getDBPath(conf, OMConfigKeys.OZONE_OM_DB_DIRS);
}
}
\ No newline at end of file
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index 9164d48..54f1452 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -661,7 +661,7 @@
// We should return only the keys, whose keys match with prefix and
// the keys after the startBucket.
- if (key.startsWith(seekPrefix) && key.compareTo(startKey) > 0) {
+ if (key.startsWith(seekPrefix) && key.compareTo(startKey) >= 0) {
result.add(omBucketInfo);
currentCount++;
}
@@ -804,6 +804,18 @@
return deletedKeys;
}
+ @Override
+ public boolean recoverTrash(String volumeName, String bucketName,
+ String keyName, String destinationBucket) throws IOException {
+
+ /* TODO: HDDS-2425 and HDDS-2426
+ core logic stub would be added in later patch.
+ */
+
+ boolean recoverOperation = true;
+ return recoverOperation;
+ }
+
/**
* @param userName volume owner, null for listing all volumes.
*/
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 627be1f..7aedef1 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -106,6 +106,7 @@
import org.apache.hadoop.ozone.om.ha.OMFailoverProxyProvider;
import org.apache.hadoop.ozone.om.ha.OMHANodeDetails;
import org.apache.hadoop.ozone.om.ha.OMNodeDetails;
+import org.apache.hadoop.ozone.om.helpers.DBUpdates;
import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
@@ -168,7 +169,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.protobuf.BlockingService;
-import org.apache.commons.codec.digest.DigestUtils;
+import com.google.protobuf.ProtocolMessageEnum;
import org.apache.commons.lang3.StringUtils;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED_DEFAULT;
@@ -185,7 +186,6 @@
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_BLOCKS_MAX;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_BLOCKS_MAX_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE;
@@ -200,14 +200,14 @@
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METRICS_SAVE_INTERVAL;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METRICS_SAVE_INTERVAL_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_VOLUME_LISTALL_ALLOWED;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_VOLUME_LISTALL_ALLOWED_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_USER_MAX_VOLUME;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_USER_MAX_VOLUME_DEFAULT;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_AUTH_METHOD;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_REQUEST;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.TOKEN_ERROR_OTHER;
-import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.S3_BUCKET_LOCK;
-import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.VOLUME_LOCK;
import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneManagerService.newReflectiveBlockingService;
import org.apache.ratis.metrics.MetricRegistries;
@@ -252,10 +252,10 @@
private BucketManager bucketManager;
private KeyManager keyManager;
private PrefixManagerImpl prefixManager;
- private S3BucketManager s3BucketManager;
private final OMMetrics metrics;
- private final ProtocolMessageMetrics omClientProtocolMetrics;
+ private final ProtocolMessageMetrics<ProtocolMessageEnum>
+ omClientProtocolMetrics;
private OzoneManagerHttpServer httpServer;
private final OMStorage omStorage;
private final ScmBlockLocationProtocol scmBlockClient;
@@ -286,12 +286,12 @@
private List<OMNodeDetails> peerNodes;
private File omRatisSnapshotDir;
private final OMRatisSnapshotInfo omRatisSnapshotInfo;
- private final Collection<String> ozAdmins;
private KeyProviderCryptoExtension kmsProvider = null;
private static String keyProviderUriKeyName =
CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH;
+ private boolean allowListAllVolumes;
// Adding parameters needed for VolumeRequests here, so that during request
// execution, we can get from ozoneManager.
private long maxUserVolumeCount;
@@ -338,6 +338,8 @@
loginOMUserIfSecurityEnabled(conf);
+ this.allowListAllVolumes = conf.getBoolean(OZONE_OM_VOLUME_LISTALL_ALLOWED,
+ OZONE_OM_VOLUME_LISTALL_ALLOWED_DEFAULT);
this.maxUserVolumeCount = conf.getInt(OZONE_OM_USER_MAX_VOLUME,
OZONE_OM_USER_MAX_VOLUME_DEFAULT);
Preconditions.checkArgument(this.maxUserVolumeCount > 0,
@@ -349,12 +351,6 @@
+ "command is executed once before starting the OM service.",
ResultCodes.OM_NOT_INITIALIZED);
}
-
- ozAdmins = conf.getTrimmedStringCollection(OZONE_ADMINISTRATORS);
- String omSPN = UserGroupInformation.getCurrentUser().getUserName();
- if (!ozAdmins.contains(omSPN)) {
- ozAdmins.add(omSPN);
- }
omMetaDir = OMStorage.getOmDbDir(configuration);
this.isAclEnabled = conf.getBoolean(OZONE_ACL_ENABLED,
OZONE_ACL_ENABLED_DEFAULT);
@@ -418,11 +414,9 @@
}
instantiateServices();
-
this.omRatisSnapshotInfo = new OMRatisSnapshotInfo(
omStorage.getCurrentDir());
initializeRatisServer();
-
if (isRatisEnabled) {
// Create Ratis storage dir
String omRatisDirectory =
@@ -443,7 +437,6 @@
}
metrics = OMMetrics.create();
-
omClientProtocolMetrics = ProtocolMessageMetrics
.create("OmClientProtocol", "Ozone Manager RPC endpoint",
OzoneManagerProtocolProtos.Type.values());
@@ -486,8 +479,6 @@
volumeManager = new VolumeManagerImpl(metadataManager, configuration);
bucketManager = new BucketManagerImpl(metadataManager, getKmsProvider(),
isRatisEnabled);
- s3BucketManager = new S3BucketManagerImpl(configuration, metadataManager,
- volumeManager, bucketManager);
if (secConfig.isSecurityEnabled()) {
s3SecretManager = new S3SecretManagerImpl(configuration, metadataManager);
delegationTokenMgr = createDelegationTokenSecretManager(configuration);
@@ -507,6 +498,7 @@
authorizer.setBucketManager(bucketManager);
authorizer.setKeyManager(keyManager);
authorizer.setPrefixManager(prefixManager);
+ authorizer.setOzoneAdmins(getOzoneAdmins(configuration));
}
} else {
accessAuthorizer = null;
@@ -655,9 +647,16 @@
OMConfigKeys.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT,
TimeUnit.MILLISECONDS);
- return new OzoneDelegationTokenSecretManager(conf, tokenMaxLifetime,
- tokenRenewInterval, tokenRemoverScanInterval, omRpcAddressTxt,
- s3SecretManager, certClient);
+ return new OzoneDelegationTokenSecretManager.Builder()
+ .setConf(conf)
+ .setTokenMaxLifetime(tokenMaxLifetime)
+ .setTokenRenewInterval(tokenRenewInterval)
+ .setTokenRemoverScanInterval(tokenRemoverScanInterval)
+ .setService(omRpcAddressTxt)
+ .setS3SecretManager(s3SecretManager)
+ .setCertificateClient(certClient)
+ .setOmServiceId(omNodeDetails.getOMServiceId())
+ .build();
}
private OzoneBlockTokenSecretManager createBlockTokenSecretManager(
@@ -1142,6 +1141,8 @@
LOG.error("OM HttpServer failed to start.", ex);
}
registerMXBean();
+
+ startJVMPauseMonitor();
setStartTime();
omState = State.RUNNING;
}
@@ -1195,10 +1196,7 @@
}
registerMXBean();
- // Start jvm monitor
- jvmPauseMonitor = new JvmPauseMonitor();
- jvmPauseMonitor.init(configuration);
- jvmPauseMonitor.start();
+ startJVMPauseMonitor();
setStartTime();
omState = State.RUNNING;
}
@@ -1301,6 +1299,9 @@
if (jvmPauseMonitor != null) {
jvmPauseMonitor.stop();
}
+ if (omSnapshotProvider != null) {
+ omSnapshotProvider.stop();
+ }
omState = State.STOPPED;
} catch (Exception e) {
LOG.error("OzoneManager stop failed.", e);
@@ -1561,7 +1562,10 @@
public void createVolume(OmVolumeArgs args) throws IOException {
try {
metrics.incNumVolumeCreates();
- checkAdmin();
+ if(isAclEnabled) {
+ checkAcls(ResourceType.VOLUME, StoreType.OZONE, ACLType.CREATE,
+ args.getVolume(), null, null);
+ }
volumeManager.createVolume(args);
AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.CREATE_VOLUME,
(args == null) ? null : args.toAuditMap()));
@@ -1585,7 +1589,7 @@
* @param vol - name of volume
* @param bucket - bucket name
* @param key - key
- * @throws OMException
+ * @throws OMException ResultCodes.PERMISSION_DENIED if permission denied.
*/
private void checkAcls(ResourceType resType, StoreType store,
ACLType acl, String vol, String bucket, String key)
@@ -1593,27 +1597,54 @@
checkAcls(resType, store, acl, vol, bucket, key,
ProtobufRpcEngine.Server.getRemoteUser(),
ProtobufRpcEngine.Server.getRemoteIp(),
- ProtobufRpcEngine.Server.getRemoteIp().getHostName());
+ ProtobufRpcEngine.Server.getRemoteIp().getHostName(),
+ true);
}
/**
+ * A variant of checkAcls that doesn't throw exception if permission denied.
+ * @return true if permission granted, false if permission denied.
+ */
+ private boolean hasAcls(ResourceType resType, StoreType store,
+ ACLType acl, String vol, String bucket, String key) {
+ try {
+ return checkAcls(resType, store, acl, vol, bucket, key,
+ ProtobufRpcEngine.Server.getRemoteUser(),
+ ProtobufRpcEngine.Server.getRemoteIp(),
+ ProtobufRpcEngine.Server.getRemoteIp().getHostName(),
+ false);
+ } catch (OMException ex) {
+ // Should not trigger exception here at all
+ return false;
+ }
+ }
+
+
+ /**
* CheckAcls for the ozone object.
- * @param resType
- * @param storeType
- * @param aclType
- * @param vol
- * @param bucket
- * @param key
- * @param ugi
- * @param remoteAddress
- * @param hostName
- * @throws OMException
+ * @throws OMException ResultCodes.PERMISSION_DENIED if permission denied.
*/
@SuppressWarnings("parameternumber")
public void checkAcls(ResourceType resType, StoreType storeType,
ACLType aclType, String vol, String bucket, String key,
UserGroupInformation ugi, InetAddress remoteAddress, String hostName)
throws OMException {
+ checkAcls(resType, storeType, aclType, vol, bucket, key,
+ ugi, remoteAddress, hostName, true);
+ }
+
+ /**
+ * CheckAcls for the ozone object.
+ * @return true if permission granted, false if permission denied.
+ * @throws OMException ResultCodes.PERMISSION_DENIED if permission denied
+ * and throwOnPermissionDenied set to true.
+ */
+ @SuppressWarnings("parameternumber")
+ private boolean checkAcls(ResourceType resType, StoreType storeType,
+ ACLType aclType, String vol, String bucket, String key,
+ UserGroupInformation ugi, InetAddress remoteAddress, String hostName,
+ boolean throwIfPermissionDenied)
+ throws OMException {
OzoneObj obj = OzoneObjInfo.Builder.newBuilder()
.setResType(resType)
.setStoreType(storeType)
@@ -1628,11 +1659,16 @@
.setAclRights(aclType)
.build();
if (!accessAuthorizer.checkAccess(obj, context)) {
- LOG.warn("User {} doesn't have {} permission to access {} /{}/{}/{}",
- ugi.getUserName(), aclType, resType, vol, bucket, key);
- throw new OMException("User " + ugi.getUserName() + " doesn't " +
- "have " + aclType + " permission to access " + resType,
- ResultCodes.PERMISSION_DENIED);
+ if (throwIfPermissionDenied) {
+ LOG.warn("User {} doesn't have {} permission to access {} /{}/{}/{}",
+ ugi.getUserName(), aclType, resType, vol, bucket, key);
+ throw new OMException("User " + ugi.getUserName() + " doesn't have " +
+ aclType + " permission to access " + resType,
+ ResultCodes.PERMISSION_DENIED);
+ }
+ return false;
+ } else {
+ return true;
}
}
@@ -1646,14 +1682,10 @@
}
/**
- * Changes the owner of a volume.
- *
- * @param volume - Name of the volume.
- * @param owner - Name of the owner.
- * @throws IOException
+ * {@inheritDoc}
*/
@Override
- public void setOwner(String volume, String owner) throws IOException {
+ public boolean setOwner(String volume, String owner) throws IOException {
if(isAclEnabled) {
checkAcls(ResourceType.VOLUME, StoreType.OZONE, ACLType.WRITE_ACL, volume,
null, null);
@@ -1665,6 +1697,7 @@
volumeManager.setOwner(volume, owner);
AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.SET_OWNER,
auditMap));
+ return true;
} catch (Exception ex) {
metrics.incNumVolumeUpdateFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.SET_OWNER,
@@ -1689,7 +1722,7 @@
}
Map<String, String> auditMap = buildAuditMap(volume);
- auditMap.put(OzoneConsts.QUOTA, String.valueOf(quota));
+ auditMap.put(OzoneConsts.QUOTA_IN_BYTES, String.valueOf(quota));
try {
metrics.incNumVolumeUpdates();
volumeManager.setQuota(volume, quota);
@@ -1715,7 +1748,7 @@
@Override
public boolean checkVolumeAccess(String volume, OzoneAclInfo userAcl)
throws IOException {
- if(isAclEnabled) {
+ if (isAclEnabled) {
checkAcls(ResourceType.VOLUME, StoreType.OZONE,
ACLType.READ, volume, null, null);
}
@@ -1800,7 +1833,7 @@
}
/**
- * Lists volume owned by a specific user.
+ * Lists volumes accessible by a specific user.
*
* @param userName - user name
* @param prefix - Filter prefix -- Return only entries that match this.
@@ -1813,13 +1846,13 @@
@Override
public List<OmVolumeArgs> listVolumeByUser(String userName, String prefix,
String prevKey, int maxKeys) throws IOException {
- if(isAclEnabled) {
- UserGroupInformation remoteUserUgi = ProtobufRpcEngine.Server.
- getRemoteUser();
+ UserGroupInformation remoteUserUgi =
+ ProtobufRpcEngine.Server.getRemoteUser();
+ if (isAclEnabled) {
if (remoteUserUgi == null) {
LOG.error("Rpc user UGI is null. Authorization failed.");
- throw new OMException("Rpc user UGI is null. Authorization " +
- "failed.", ResultCodes.PERMISSION_DENIED);
+ throw new OMException("Rpc user UGI is null. Authorization failed.",
+ ResultCodes.PERMISSION_DENIED);
}
}
boolean auditSuccess = true;
@@ -1830,7 +1863,23 @@
auditMap.put(OzoneConsts.USERNAME, userName);
try {
metrics.incNumVolumeLists();
- return volumeManager.listVolumes(userName, prefix, prevKey, maxKeys);
+ if (isAclEnabled) {
+ // List all volumes first
+ List<OmVolumeArgs> listAllVolumes = volumeManager.listVolumes(
+ null, prefix, prevKey, maxKeys);
+ List<OmVolumeArgs> result = new ArrayList<>();
+ // Filter all volumes by LIST ACL
+ for (OmVolumeArgs volumeArgs : listAllVolumes) {
+ if (hasAcls(ResourceType.VOLUME, StoreType.OZONE, ACLType.LIST,
+ volumeArgs.getVolume(), null, null)) {
+ result.add(volumeArgs);
+ }
+ }
+ return result;
+ } else {
+ // When ACL is not enabled, fallback to filter by owner
+ return volumeManager.listVolumes(userName, prefix, prevKey, maxKeys);
+ }
} catch (Exception ex) {
metrics.incNumVolumeListFails();
auditSuccess = false;
@@ -1866,7 +1915,13 @@
auditMap.put(OzoneConsts.USERNAME, null);
try {
metrics.incNumVolumeLists();
- checkAdmin();
+ if (!allowListAllVolumes) {
+ // Only admin can list all volumes when disallowed in config
+ if(isAclEnabled) {
+ checkAcls(ResourceType.VOLUME, StoreType.OZONE, ACLType.LIST,
+ OzoneConsts.OZONE_ROOT, null, null);
+ }
+ }
return volumeManager.listVolumes(null, prefix, prevKey, maxKeys);
} catch (Exception ex) {
metrics.incNumVolumeListFails();
@@ -1882,20 +1937,6 @@
}
}
- private void checkAdmin() throws OMException {
- if(isAclEnabled) {
- if (!ozAdmins.contains(OZONE_ADMINISTRATORS_WILDCARD) &&
- !ozAdmins.contains(ProtobufRpcEngine.Server.getRemoteUser()
- .getUserName())) {
- LOG.error("Only admin users are authorized to create or list " +
- "Ozone volumes. User :{} is not an admin.",
- ProtobufRpcEngine.Server.getRemoteUser().getUserName());
- throw new OMException("Only admin users are authorized to create " +
- "or list Ozone volumes.", ResultCodes.PERMISSION_DENIED);
- }
- }
- }
-
/**
* Creates a bucket.
*
@@ -2262,15 +2303,6 @@
}
}
- // TODO: HDDS-2424. recover-trash command server side handling.
- @Override
- public boolean recoverTrash(String volumeName, String bucketName,
- String keyName, String destinationBucket) throws IOException {
-
- boolean recoverOperation = true;
- return recoverOperation;
- }
-
/**
* Sets bucket property from args.
*
@@ -2508,70 +2540,6 @@
/**
* {@inheritDoc}
*/
- public void createS3Bucket(String userName, String s3BucketName)
- throws IOException {
-
- boolean acquiredS3Lock = false;
- boolean acquiredVolumeLock = false;
- try {
- metrics.incNumBucketCreates();
- acquiredS3Lock = metadataManager.getLock().acquireLock(S3_BUCKET_LOCK,
- s3BucketName);
- try {
- acquiredVolumeLock = metadataManager.getLock().acquireLock(VOLUME_LOCK,
- s3BucketManager.formatOzoneVolumeName(userName));
- boolean newVolumeCreate = s3BucketManager.createOzoneVolumeIfNeeded(
- userName);
- if (newVolumeCreate) {
- metrics.incNumVolumeCreates();
- metrics.incNumVolumes();
- }
- } catch (IOException ex) {
- // We need to increment volume creates also because this is first
- // time we are trying to create a volume, it failed. As we increment
- // ops and create when we try to do that operation.
- metrics.incNumVolumeCreates();
- metrics.incNumVolumeCreateFails();
- throw ex;
- }
- s3BucketManager.createS3Bucket(userName, s3BucketName);
- metrics.incNumBuckets();
- } catch (IOException ex) {
- metrics.incNumBucketCreateFails();
- throw ex;
- } finally {
- if (acquiredVolumeLock) {
- metadataManager.getLock().releaseLock(VOLUME_LOCK,
- s3BucketManager.formatOzoneVolumeName(userName));
- }
- if (acquiredS3Lock) {
- metadataManager.getLock().releaseLock(S3_BUCKET_LOCK, s3BucketName);
- }
- }
- }
-
- @Override
- /**
- * {@inheritDoc}
- */
- public void deleteS3Bucket(String s3BucketName) throws IOException {
- try {
- if(isAclEnabled) {
- checkAcls(ResourceType.BUCKET, StoreType.S3, ACLType.DELETE,
- getS3VolumeName(), s3BucketName, null);
- }
- metrics.incNumBucketDeletes();
- s3BucketManager.deleteS3Bucket(s3BucketName);
- metrics.decNumBuckets();
- } catch (IOException ex) {
- metrics.incNumBucketDeleteFails();
- }
- }
-
- @Override
- /**
- * {@inheritDoc}
- */
public S3SecretValue getS3Secret(String kerberosID) throws IOException{
UserGroupInformation user = ProtobufRpcEngine.Server.getRemoteUser();
@@ -2585,59 +2553,6 @@
}
@Override
- /**
- * {@inheritDoc}
- */
- public String getOzoneBucketMapping(String s3BucketName)
- throws IOException {
- if(isAclEnabled) {
- checkAcls(ResourceType.BUCKET, StoreType.S3, ACLType.READ,
- getS3VolumeName(), s3BucketName, null);
- }
- return s3BucketManager.getOzoneBucketMapping(s3BucketName);
- }
-
- /**
- * Helper function to return volume name for S3 users.
- * */
- private String getS3VolumeName() {
- return s3BucketManager.formatOzoneVolumeName(DigestUtils.md5Hex(
- ProtobufRpcEngine.Server.getRemoteUser().getUserName().toLowerCase()));
- }
-
- @Override
- public List<OmBucketInfo> listS3Buckets(String userName, String startKey,
- String prefix, int maxNumOfBuckets)
- throws IOException {
- if(isAclEnabled) {
- checkAcls(ResourceType.VOLUME, StoreType.S3, ACLType.LIST,
- s3BucketManager.getOzoneVolumeNameForUser(userName), null, null);
- }
- boolean auditSuccess = true;
- Map<String, String> auditMap = buildAuditMap(userName);
- auditMap.put(OzoneConsts.START_KEY, startKey);
- auditMap.put(OzoneConsts.PREFIX, prefix);
- auditMap.put(OzoneConsts.MAX_NUM_OF_BUCKETS,
- String.valueOf(maxNumOfBuckets));
- try {
- metrics.incNumListS3Buckets();
- String volumeName = s3BucketManager.getOzoneVolumeNameForUser(userName);
- return bucketManager.listBuckets(volumeName, startKey, prefix,
- maxNumOfBuckets);
- } catch (IOException ex) {
- metrics.incNumListS3BucketsFails();
- auditSuccess = false;
- AUDIT.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_S3BUCKETS,
- auditMap, ex));
- throw ex;
- } finally {
- if(auditSuccess){
- AUDIT.logReadSuccess(buildAuditMessageForSuccess(OMAction
- .LIST_S3BUCKETS, auditMap));
- }
- }
- }
- @Override
public OmMultipartInfo initiateMultipartUpload(OmKeyArgs keyArgs) throws
IOException {
OmMultipartInfo multipartInfo;
@@ -3352,12 +3267,14 @@
* @throws SequenceNumberNotFoundException if db is unable to read the data.
*/
@Override
- public DBUpdatesWrapper getDBUpdates(
+ public DBUpdates getDBUpdates(
DBUpdatesRequest dbUpdatesRequest)
throws SequenceNumberNotFoundException {
- return metadataManager.getStore()
+ DBUpdatesWrapper updatesSince = metadataManager.getStore()
.getUpdatesSince(dbUpdatesRequest.getSequenceNumber());
-
+ DBUpdates dbUpdates = new DBUpdates(updatesSince.getData());
+ dbUpdates.setCurrentSequenceNumber(updatesSince.getCurrentSequenceNumber());
+ return dbUpdates;
}
public OzoneDelegationTokenSecretManager getDelegationTokenMgr() {
@@ -3367,7 +3284,14 @@
/**
* Return list of OzoneAdministrators.
*/
- public Collection<String> getOzoneAdmins() {
+ private Collection<String> getOzoneAdmins(OzoneConfiguration conf)
+ throws IOException {
+ Collection<String> ozAdmins =
+ conf.getTrimmedStringCollection(OZONE_ADMINISTRATORS);
+ String omSPN = UserGroupInformation.getCurrentUser().getShortUserName();
+ if (!ozAdmins.contains(omSPN)) {
+ ozAdmins.add(omSPN);
+ }
return ozAdmins;
}
@@ -3383,4 +3307,11 @@
public boolean isRunning() {
return omState == State.RUNNING;
}
+
+ private void startJVMPauseMonitor() {
+ // Start jvm monitor
+ jvmPauseMonitor = new JvmPauseMonitor();
+ jvmPauseMonitor.init(configuration);
+ jvmPauseMonitor.start();
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
index 38dc1ad..cd1c085 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
@@ -17,12 +17,12 @@
package org.apache.hadoop.ozone.om;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.hdds.server.http.BaseHttpServer;
-
import java.io.IOException;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.server.http.BaseHttpServer;
+import org.apache.hadoop.ozone.OzoneConsts;
+
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OM_DB_CHECKPOINT_HTTP_ENDPOINT;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OM_SERVICE_LIST_HTTP_ENDPOINT;
@@ -31,7 +31,7 @@
*/
public class OzoneManagerHttpServer extends BaseHttpServer {
- public OzoneManagerHttpServer(Configuration conf, OzoneManager om)
+ public OzoneManagerHttpServer(ConfigurationSource conf, OzoneManager om)
throws IOException {
super(conf, "ozoneManager");
addServlet("serviceList", OZONE_OM_SERVICE_LIST_HTTP_ENDPOINT,
@@ -80,4 +80,14 @@
@Override protected String getEnabledKey() {
return OMConfigKeys.OZONE_OM_HTTP_ENABLED_KEY;
}
+
+ @Override
+ protected String getHttpAuthType() {
+ return OMConfigKeys.OZONE_OM_HTTP_AUTH_TYPE;
+ }
+
+ @Override
+ protected String getHttpAuthConfigPrefix() {
+ return OMConfigKeys.OZONE_OM_HTTP_AUTH_CONFIG_PREFIX;
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java
deleted file mode 100644
index dfd0ac3..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.hadoop.ozone.om;
-
-import java.io.IOException;
-
-/**
- * An interface that maps S3 buckets to Ozone
- * volume/bucket.
- */
-public interface S3BucketManager {
- /**
- * Creates an s3 bucket and maps it to Ozone volume/bucket.
- * @param userName - Name of the user who owns the bucket.
- * @param bucketName - S3 Bucket Name.
- * @throws IOException in case the bucket cannot be created.
- */
- void createS3Bucket(String userName, String bucketName) throws IOException;
-
- /**
- * Deletes an s3 bucket and removes mapping of Ozone volume/bucket.
- * @param bucketName - S3 Bucket Name.
- * @throws IOException in case the bucket cannot be deleted.
- */
- void deleteS3Bucket(String bucketName) throws IOException;
-
- /**
- * Returns the Ozone volume/bucket where the S3 Bucket points to.
- * @param s3BucketName - S3 Bucket Name
- * @return String - Ozone volume/bucket
- * @throws IOException in case of failure to retrieve mapping.
- */
- String getOzoneBucketMapping(String s3BucketName) throws IOException;
-
- /**
- * Returns Ozone volume name for a given S3Bucket.
- * @param s3BucketName - S3 bucket name.
- * @return String - Ozone volume name where is s3bucket resides.
- * @throws IOException - in case of failure to retrieve mapping.
- */
- String getOzoneVolumeName(String s3BucketName) throws IOException;
-
- /**
- * Returns Ozone bucket name for a given s3Bucket.
- * @param s3BucketName - S3 bucket Name.
- * @return Ozone bucket name for this given S3 bucket
- * @throws IOException - in case of failure to retrieve mapping.
- */
- String getOzoneBucketName(String s3BucketName) throws IOException;
-
- /**
- * Returns volume Name for a user.
- * @param userName
- */
- String getOzoneVolumeNameForUser(String userName) throws IOException;
-
- /**
- * Create ozone volume if required, this will be needed during creates3Bucket.
- * @param userName
- * @return true - if volume is successfully created. false - if volume
- * already exists or volume creation failure.
- * @throws IOException - incase of volume creation failure.
- */
- boolean createOzoneVolumeIfNeeded(String userName) throws IOException;
-
- /**
- * Return volume name from userName.
- * @param userName
- */
- String formatOzoneVolumeName(String userName);
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java
deleted file mode 100644
index 8a581bb..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java
+++ /dev/null
@@ -1,254 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.hadoop.ozone.om;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.StorageType;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.om.exceptions.OMException;
-import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
-import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
-import org.apache.hadoop.security.UserGroupInformation;
-
-import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_ALREADY_EXISTS;
-
-import org.apache.logging.log4j.util.Strings;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Objects;
-
-import static org.apache.hadoop.ozone.OzoneConsts.OM_S3_VOLUME_PREFIX;
-import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.S3_BUCKET_LOCK;
-
-/**
- * S3 Bucket Manager, this class maintains a mapping between S3 Bucket and Ozone
- * Volume/bucket.
- */
-public class S3BucketManagerImpl implements S3BucketManager {
- private static final Logger LOG =
- LoggerFactory.getLogger(S3BucketManagerImpl.class);
-
- private static final String S3_ADMIN_NAME = "OzoneS3Manager";
- private final OzoneConfiguration configuration;
- private final OMMetadataManager omMetadataManager;
- private final VolumeManager volumeManager;
- private final BucketManager bucketManager;
-
- /**
- * Construct an S3 Bucket Manager Object.
- *
- * @param configuration - Ozone Configuration.
- * @param omMetadataManager - Ozone Metadata Manager.
- */
- public S3BucketManagerImpl(
- OzoneConfiguration configuration,
- OMMetadataManager omMetadataManager,
- VolumeManager volumeManager,
- BucketManager bucketManager) {
- this.configuration = configuration;
- this.omMetadataManager = omMetadataManager;
- this.volumeManager = volumeManager;
- this.bucketManager = bucketManager;
- }
-
- @Override
- public void createS3Bucket(String userName, String bucketName)
- throws IOException {
- Preconditions.checkArgument(Strings.isNotBlank(bucketName), "Bucket" +
- " name cannot be null or empty.");
- Preconditions.checkArgument(Strings.isNotBlank(userName), "User name " +
- "cannot be null or empty.");
-
- Preconditions.checkArgument(bucketName.length() >=3 &&
- bucketName.length() < 64, "Length of the S3 Bucket is not correct.");
-
-
- // TODO: Decide if we want to enforce S3 Bucket Creation Rules in this
- // code path?
- // https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html
-
- // Generate an Ozone volume name. For the time being, we are going to use
- // s3userName as the Ozone volume name. Since S3 advices 100 buckets max
- // for a user and we have no limit to the number of Ozone buckets under a
- // volume we will stick to very simple model.
- //
- // s3Bucket -> ozoneVolume/OzoneBucket name
- // s3BucketName ->s3userName/s3Bucketname
- //
- // You might wonder if all names map to this pattern, why we need to
- // store the S3 bucketName in a table at all. This is to support
- // anonymous access to bucket where the user name is absent.
- String ozoneVolumeName = formatOzoneVolumeName(userName);
-
- omMetadataManager.getLock().acquireLock(S3_BUCKET_LOCK, bucketName);
- try {
- String bucket = omMetadataManager.getS3Table().get(bucketName);
-
- if (bucket != null) {
- LOG.debug("Bucket already exists. {}", bucketName);
- throw new OMException(
- "Unable to create S3 bucket. " + bucketName + " already exists.",
- OMException.ResultCodes.S3_BUCKET_ALREADY_EXISTS);
- }
- String ozoneBucketName = bucketName;
- createOzoneBucket(ozoneVolumeName, ozoneBucketName);
- String finalName = String.format("%s/%s", ozoneVolumeName,
- ozoneBucketName);
-
- omMetadataManager.getS3Table().put(bucketName, finalName);
- } finally {
- omMetadataManager.getLock().releaseLock(S3_BUCKET_LOCK, bucketName);
- }
-
- }
-
- @Override
- public void deleteS3Bucket(String bucketName) throws IOException {
- Preconditions.checkArgument(
- Strings.isNotBlank(bucketName), "Bucket name cannot be null or empty");
-
- omMetadataManager.getLock().acquireLock(S3_BUCKET_LOCK, bucketName);
- try {
- String map = omMetadataManager.getS3Table().get(bucketName);
-
- if (map == null) {
- throw new OMException("No such S3 bucket. " + bucketName,
- OMException.ResultCodes.S3_BUCKET_NOT_FOUND);
- }
-
- bucketManager.deleteBucket(getOzoneVolumeName(bucketName), bucketName);
- omMetadataManager.getS3Table().delete(bucketName);
- } catch(IOException ex) {
- throw ex;
- } finally {
- omMetadataManager.getLock().releaseLock(S3_BUCKET_LOCK, bucketName);
- }
-
- }
-
- @Override
- public String formatOzoneVolumeName(String userName) {
- return String.format(OM_S3_VOLUME_PREFIX + "%s", userName);
- }
-
- @Override
- public boolean createOzoneVolumeIfNeeded(String userName)
- throws IOException {
- // We don't have to time of check. time of use problem here because
- // this call is invoked while holding the s3Bucket lock.
- boolean newVolumeCreate = true;
- String ozoneVolumeName = formatOzoneVolumeName(userName);
- try {
- OmVolumeArgs.Builder builder =
- OmVolumeArgs.newBuilder()
- .setAdminName(S3_ADMIN_NAME)
- .setOwnerName(userName)
- .setVolume(ozoneVolumeName)
- .setQuotaInBytes(OzoneConsts.MAX_QUOTA_IN_BYTES);
- for (OzoneAcl acl : getDefaultAcls(userName)) {
- builder.addOzoneAcls(OzoneAcl.toProtobuf(acl));
- }
-
- OmVolumeArgs args = builder.build();
-
- volumeManager.createVolume(args);
-
- } catch (OMException exp) {
- newVolumeCreate = false;
- if (exp.getResult().compareTo(VOLUME_ALREADY_EXISTS) == 0) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Volume already exists. {}", exp.getMessage());
- }
- } else {
- throw exp;
- }
- }
-
- return newVolumeCreate;
- }
-
- /**
- * Get default acls.
- * */
- private List<OzoneAcl> getDefaultAcls(String userName) {
- UserGroupInformation ugi = ProtobufRpcEngine.Server.getRemoteUser();
- return OzoneAcl.parseAcls("user:" + (ugi == null ? userName :
- ugi.getUserName()) + ":a,user:" + S3_ADMIN_NAME + ":a");
- }
-
- private void createOzoneBucket(String volumeName, String bucketName)
- throws IOException {
- OmBucketInfo.Builder builder = OmBucketInfo.newBuilder();
- OmBucketInfo bucketInfo =
- builder
- .setVolumeName(volumeName)
- .setBucketName(bucketName)
- .setIsVersionEnabled(Boolean.FALSE)
- .setStorageType(StorageType.DEFAULT)
- .setAcls(getDefaultAcls(null))
- .build();
- bucketManager.createBucket(bucketInfo);
- }
-
- @Override
- public String getOzoneBucketMapping(String s3BucketName) throws IOException {
- Preconditions.checkArgument(
- Strings.isNotBlank(s3BucketName),
- "Bucket name cannot be null or empty.");
- Preconditions.checkArgument(s3BucketName.length() >=3 &&
- s3BucketName.length() < 64,
- "Length of the S3 Bucket is not correct.");
- omMetadataManager.getLock().acquireLock(S3_BUCKET_LOCK, s3BucketName);
- try {
- String mapping = omMetadataManager.getS3Table().get(s3BucketName);
- if (mapping != null) {
- return mapping;
- }
- throw new OMException("No such S3 bucket.",
- OMException.ResultCodes.S3_BUCKET_NOT_FOUND);
- } finally {
- omMetadataManager.getLock().releaseLock(S3_BUCKET_LOCK, s3BucketName);
- }
- }
-
- @Override
- public String getOzoneVolumeName(String s3BucketName) throws IOException {
- String mapping = getOzoneBucketMapping(s3BucketName);
- return mapping.split("/")[0];
- }
-
- @Override
- public String getOzoneBucketName(String s3BucketName) throws IOException {
- String mapping = getOzoneBucketMapping(s3BucketName);
- return mapping.split("/")[1];
- }
-
- @Override
- public String getOzoneVolumeNameForUser(String userName) throws IOException {
- Objects.requireNonNull(userName, "UserName cannot be null");
- return formatOzoneVolumeName(userName);
- }
-
-}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/S3SecretManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3SecretManagerImpl.java
similarity index 100%
rename from hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/S3SecretManagerImpl.java
rename to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3SecretManagerImpl.java
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeManagerImpl.java
index 9d9cdab..f7d730a 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeManagerImpl.java
@@ -20,10 +20,8 @@
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
-import java.util.stream.Collectors;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.om.exceptions.OMException;
@@ -32,10 +30,8 @@
.OzoneManagerProtocolProtos.OzoneAclInfo;
import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.UserVolumeInfo;
-import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.apache.hadoop.ozone.security.acl.RequestContext;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.hdds.utils.db.BatchOperation;
import com.google.common.base.Preconditions;
@@ -453,23 +449,11 @@
@Override
public List<OmVolumeArgs> listVolumes(String userName,
String prefix, String startKey, int maxKeys) throws IOException {
- metadataManager.getLock().acquireLock(USER_LOCK, userName);
+ metadataManager.getLock().acquireReadLock(USER_LOCK, userName);
try {
- List<OmVolumeArgs> volumes = metadataManager.listVolumes(
- userName, prefix, startKey, maxKeys);
- UserGroupInformation userUgi = ProtobufRpcEngine.Server.
- getRemoteUser();
- if (userUgi == null || !aclEnabled) {
- return volumes;
- }
-
- List<OmVolumeArgs> filteredVolumes = volumes.stream().
- filter(v -> v.getAclMap().
- hasAccess(IAccessAuthorizer.ACLType.LIST, userUgi))
- .collect(Collectors.toList());
- return filteredVolumes;
+ return metadataManager.listVolumes(userName, prefix, startKey, maxKeys);
} finally {
- metadataManager.getLock().releaseLock(USER_LOCK, userName);
+ metadataManager.getLock().releaseReadLock(USER_LOCK, userName);
}
}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/OmBucketInfoCodec.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OmBucketInfoCodec.java
similarity index 100%
rename from hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/OmBucketInfoCodec.java
rename to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OmBucketInfoCodec.java
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/OmKeyInfoCodec.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OmKeyInfoCodec.java
similarity index 100%
rename from hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/OmKeyInfoCodec.java
rename to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OmKeyInfoCodec.java
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/OmMultipartKeyInfoCodec.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OmMultipartKeyInfoCodec.java
similarity index 100%
rename from hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/OmMultipartKeyInfoCodec.java
rename to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OmMultipartKeyInfoCodec.java
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/OmPrefixInfoCodec.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OmPrefixInfoCodec.java
similarity index 100%
rename from hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/OmPrefixInfoCodec.java
rename to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OmPrefixInfoCodec.java
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/OmVolumeArgsCodec.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OmVolumeArgsCodec.java
similarity index 100%
rename from hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/OmVolumeArgsCodec.java
rename to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OmVolumeArgsCodec.java
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/RepeatedOmKeyInfoCodec.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/RepeatedOmKeyInfoCodec.java
similarity index 100%
rename from hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/RepeatedOmKeyInfoCodec.java
rename to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/RepeatedOmKeyInfoCodec.java
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/S3SecretValueCodec.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/S3SecretValueCodec.java
similarity index 100%
rename from hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/S3SecretValueCodec.java
rename to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/S3SecretValueCodec.java
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/TokenIdentifierCodec.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/TokenIdentifierCodec.java
similarity index 100%
rename from hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/TokenIdentifierCodec.java
rename to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/TokenIdentifierCodec.java
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/UserVolumeInfoCodec.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/UserVolumeInfoCodec.java
similarity index 100%
rename from hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/UserVolumeInfoCodec.java
rename to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/UserVolumeInfoCodec.java
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/package-info.java
similarity index 100%
rename from hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/package-info.java
rename to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/package-info.java
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java
index a64e53a..c968efb 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java
@@ -25,9 +25,13 @@
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hdds.function.SupplierWithIOException;
+import org.apache.hadoop.hdds.tracing.TracingUtil;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,7 +54,7 @@
* methods.
*
*/
-public class OzoneManagerDoubleBuffer {
+public final class OzoneManagerDoubleBuffer {
private static final Logger LOG =
LoggerFactory.getLogger(OzoneManagerDoubleBuffer.class);
@@ -85,20 +89,52 @@
private final OzoneManagerRatisSnapshot ozoneManagerRatisSnapShot;
private final boolean isRatisEnabled;
+ private final boolean isTracingEnabled;
- public OzoneManagerDoubleBuffer(OMMetadataManager omMetadataManager,
- OzoneManagerRatisSnapshot ozoneManagerRatisSnapShot) {
- this(omMetadataManager, ozoneManagerRatisSnapShot, true);
+ /**
+ * Builder for creating OzoneManagerDoubleBuffer.
+ */
+ public static class Builder {
+ private OMMetadataManager mm;
+ private OzoneManagerRatisSnapshot rs;
+ private boolean isRatisEnabled = false;
+ private boolean isTracingEnabled = false;
+
+ public Builder setOmMetadataManager(OMMetadataManager omm) {
+ this.mm = omm;
+ return this;
+ }
+
+ public Builder setOzoneManagerRatisSnapShot(
+ OzoneManagerRatisSnapshot omrs) {
+ this.rs = omrs;
+ return this;
+ }
+
+ public Builder enableRatis(boolean enableRatis) {
+ this.isRatisEnabled = enableRatis;
+ return this;
+ }
+
+ public Builder enableTracing(boolean enableTracing) {
+ this.isTracingEnabled = enableTracing;
+ return this;
+ }
+
+ public OzoneManagerDoubleBuffer build() {
+ return new OzoneManagerDoubleBuffer(mm, rs, isRatisEnabled,
+ isTracingEnabled);
+ }
}
- public OzoneManagerDoubleBuffer(OMMetadataManager omMetadataManager,
+ private OzoneManagerDoubleBuffer(OMMetadataManager omMetadataManager,
OzoneManagerRatisSnapshot ozoneManagerRatisSnapShot,
- boolean isRatisEnabled) {
+ boolean isRatisEnabled, boolean isTracingEnabled) {
this.currentBuffer = new ConcurrentLinkedQueue<>();
this.readyBuffer = new ConcurrentLinkedQueue<>();
this.isRatisEnabled = isRatisEnabled;
-
+ this.isTracingEnabled = isTracingEnabled;
if (!isRatisEnabled) {
this.currentFutureQueue = new ConcurrentLinkedQueue<>();
this.readyFutureQueue = new ConcurrentLinkedQueue<>();
@@ -120,8 +156,33 @@
}
+ // TODO: pass the trace id further down and trace all methods of DBStore.
+ /**
+ * add to write batch with trace span if tracing is enabled.
+ */
+ private Void addToBatchWithTrace(OMResponse omResponse,
+ SupplierWithIOException<Void> supplier) throws IOException {
+ if (!isTracingEnabled) {
+ return supplier.get();
+ }
+ String spanName = "DB-addToWriteBatch" + "-" +
+ omResponse.getCmdType().toString();
+ return TracingUtil.executeAsChildSpan(spanName, omResponse.getTraceID(),
+ supplier);
+ }
+ /**
+ * flush write batch with trace span if tracing is enabled.
+ */
+ private Void flushBatchWithTrace(String parentName, int batchSize,
+ SupplierWithIOException<Void> supplier) throws IOException {
+ if (!isTracingEnabled) {
+ return supplier.get();
+ }
+ String spanName = "DB-commitWriteBatch-Size-" + batchSize;
+ return TracingUtil.executeAsChildSpan(spanName, parentName, supplier);
+ }
/**
* Runs in a background thread and batches the transaction in currentBuffer
@@ -135,10 +196,17 @@
try(BatchOperation batchOperation = omMetadataManager.getStore()
.initBatchOperation()) {
+ AtomicReference<String> lastTraceId = new AtomicReference<>();
readyBuffer.iterator().forEachRemaining((entry) -> {
try {
- entry.getResponse().checkAndUpdateDB(omMetadataManager,
- batchOperation);
+ OMResponse omResponse = entry.getResponse().getOMResponse();
+ lastTraceId.set(omResponse.getTraceID());
+ addToBatchWithTrace(omResponse,
+ (SupplierWithIOException<Void>) () -> {
+ entry.getResponse().checkAndUpdateDB(omMetadataManager,
+ batchOperation);
+ return null;
+ });
} catch (IOException ex) {
// During Adding to RocksDB batch entry got an exception.
// We should terminate the OM.
@@ -147,7 +215,12 @@
});
long startTime = Time.monotonicNowNanos();
- omMetadataManager.getStore().commitBatchOperation(batchOperation);
+ flushBatchWithTrace(lastTraceId.get(), readyBuffer.size(),
+ (SupplierWithIOException<Void>) () -> {
+ omMetadataManager.getStore().commitBatchOperation(
+ batchOperation);
+ return null;
+ });
ozoneManagerDoubleBufferMetrics.updateFlushTime(
Time.monotonicNowNanos() - startTime);
}
@@ -173,10 +246,6 @@
flushedTransactionsSize);
}
- long lastRatisTransactionIndex =
- readyBuffer.stream().map(DoubleBufferEntry::getTrxLogIndex)
- .max(Long::compareTo).get();
-
List<Long> flushedEpochs =
readyBuffer.stream().map(DoubleBufferEntry::getTrxLogIndex)
.sorted().collect(Collectors.toList());
@@ -267,6 +336,9 @@
/**
* Stop OM DoubleBuffer flush thread.
*/
+ // Ignore the sonar false positive on the InterruptedException issue
+ // as this a normal flow of a shutdown.
+ @SuppressWarnings("squid:S2142")
public void stop() {
if (isRunning.compareAndSet(true, false)) {
LOG.info("Stopping OMDoubleBuffer flush thread");
@@ -275,7 +347,7 @@
// Wait for daemon thread to exit
daemon.join();
} catch (InterruptedException e) {
- LOG.error("Interrupted while waiting for daemon to exit.");
+ LOG.debug("Interrupted while waiting for daemon to exit.", e);
}
// stop metrics.
@@ -368,5 +440,4 @@
return ozoneManagerDoubleBufferMetrics;
}
-}
-
+}
\ No newline at end of file
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java
index 17585d4..359ab1f 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.ozone.om.ratis;
-import com.google.common.annotations.VisibleForTesting;
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -35,30 +34,31 @@
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import com.google.common.base.Strings;
-import com.google.protobuf.InvalidProtocolBufferException;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.StorageUnit;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.server.ServerUtils;
+import org.apache.hadoop.hdds.tracing.TracingUtil;
import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.exceptions.OMLeaderNotReadyException;
import org.apache.hadoop.ozone.om.exceptions.OMNotLeaderException;
import org.apache.hadoop.ozone.om.ha.OMNodeDetails;
-import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.helpers.OMRatisHelper;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
- .OMRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
- .OMResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
+import com.google.protobuf.InvalidProtocolBufferException;
+import com.google.protobuf.ServiceException;
import org.apache.ratis.RaftConfigKeys;
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.grpc.GrpcConfigKeys;
import org.apache.ratis.netty.NettyConfigKeys;
-import org.apache.ratis.proto.RaftProtos.RoleInfoProto;
import org.apache.ratis.proto.RaftProtos.RaftPeerRole;
+import org.apache.ratis.proto.RaftProtos.RoleInfoProto;
import org.apache.ratis.protocol.ClientId;
import org.apache.ratis.protocol.GroupInfoReply;
import org.apache.ratis.protocol.GroupInfoRequest;
@@ -180,9 +180,10 @@
StateMachineException stateMachineException =
reply.getStateMachineException();
if (stateMachineException != null) {
- OMResponse.Builder omResponse = OMResponse.newBuilder();
- omResponse.setCmdType(omRequest.getCmdType());
- omResponse.setSuccess(false);
+ OMResponse.Builder omResponse = OMResponse.newBuilder()
+ .setCmdType(omRequest.getCmdType())
+ .setSuccess(false)
+ .setTraceID(omRequest.getTraceID());
if (stateMachineException.getCause() != null) {
omResponse.setMessage(stateMachineException.getCause().getMessage());
omResponse.setStatus(
@@ -245,7 +246,7 @@
* @param raftPeers peer nodes in the raft ring
* @throws IOException
*/
- private OzoneManagerRatisServer(Configuration conf,
+ private OzoneManagerRatisServer(ConfigurationSource conf,
OzoneManager om,
String raftGroupIdStr, RaftPeerId localRaftPeerId,
InetSocketAddress addr, List<RaftPeer> raftPeers)
@@ -267,7 +268,7 @@
LOG.info("Instantiating OM Ratis server with GroupID: {} and " +
"Raft Peers: {}", raftGroupIdStr, raftPeersStr.toString().substring(2));
- this.omStateMachine = getStateMachine();
+ this.omStateMachine = getStateMachine(conf);
this.server = RaftServer.newBuilder()
.setServerId(this.raftPeerId)
@@ -295,7 +296,7 @@
* Creates an instance of OzoneManagerRatisServer.
*/
public static OzoneManagerRatisServer newOMRatisServer(
- Configuration ozoneConf, OzoneManager omProtocol,
+ ConfigurationSource ozoneConf, OzoneManager omProtocol,
OMNodeDetails omNodeDetails, List<OMNodeDetails> peerNodes)
throws IOException {
@@ -336,8 +337,9 @@
/**
* Initializes and returns OzoneManager StateMachine.
*/
- private OzoneManagerStateMachine getStateMachine() {
- return new OzoneManagerStateMachine(this);
+ private OzoneManagerStateMachine getStateMachine(ConfigurationSource conf) {
+ return new OzoneManagerStateMachine(this,
+ TracingUtil.isTracingEnabled(conf));
}
@VisibleForTesting
@@ -370,7 +372,7 @@
//TODO simplify it to make it shorter
@SuppressWarnings("methodlength")
- private RaftProperties newRaftProperties(Configuration conf) {
+ private RaftProperties newRaftProperties(ConfigurationSource conf) {
final RaftProperties properties = new RaftProperties();
// Set RPC type
@@ -389,7 +391,7 @@
// Set Ratis storage directory
String storageDir = OzoneManagerRatisServer.getOMRatisDirectory(conf);
- RaftServerConfigKeys.setStorageDirs(properties,
+ RaftServerConfigKeys.setStorageDir(properties,
Collections.singletonList(new File(storageDir)));
// Set RAFT segment size
@@ -473,7 +475,7 @@
serverMaxTimeout);
// Set the number of maximum cached segments
- RaftServerConfigKeys.Log.setMaxCachedSegmentNum(properties, 2);
+ RaftServerConfigKeys.Log.setSegmentCacheNumMax(properties, 2);
// TODO: set max write buffer size
@@ -657,7 +659,7 @@
/**
* Get the local directory where ratis logs will be stored.
*/
- public static String getOMRatisDirectory(Configuration conf) {
+ public static String getOMRatisDirectory(ConfigurationSource conf) {
String storageDir = conf.get(OMConfigKeys.OZONE_OM_RATIS_STORAGE_DIR);
if (Strings.isNullOrEmpty(storageDir)) {
@@ -666,7 +668,7 @@
return storageDir;
}
- public static String getOMRatisSnapshotDirectory(Configuration conf) {
+ public static String getOMRatisSnapshotDirectory(ConfigurationSource conf) {
String snapshotDir = conf.get(OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_DIR);
if (Strings.isNullOrEmpty(snapshotDir)) {
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java
index 5ac17bf..3cf22f6 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java
@@ -85,6 +85,7 @@
private final OMRatisSnapshotInfo snapshotInfo;
private final ExecutorService executorService;
private final ExecutorService installSnapshotExecutor;
+ private final boolean isTracingEnabled;
// Map which contains index and term for the ratis transactions which are
// stateMachine entries which are recived through applyTransaction.
@@ -97,16 +98,21 @@
new ConcurrentSkipListMap<>();
- public OzoneManagerStateMachine(OzoneManagerRatisServer ratisServer) {
+ public OzoneManagerStateMachine(OzoneManagerRatisServer ratisServer,
+ boolean isTracingEnabled) {
this.omRatisServer = ratisServer;
+ this.isTracingEnabled = isTracingEnabled;
this.ozoneManager = omRatisServer.getOzoneManager();
this.snapshotInfo = ozoneManager.getSnapshotInfo();
updateLastAppliedIndexWithSnaphsotIndex();
- this.ozoneManagerDoubleBuffer =
- new OzoneManagerDoubleBuffer(ozoneManager.getMetadataManager(),
- this::updateLastAppliedIndex);
+ this.ozoneManagerDoubleBuffer = new OzoneManagerDoubleBuffer.Builder()
+ .setOmMetadataManager(ozoneManager.getMetadataManager())
+ .setOzoneManagerRatisSnapShot(this::updateLastAppliedIndex)
+ .enableRatis(true)
+ .enableTracing(isTracingEnabled)
+ .build();
this.handler = new OzoneManagerRequestHandler(ozoneManager,
ozoneManagerDoubleBuffer);
@@ -123,7 +129,7 @@
@Override
public void initialize(RaftServer server, RaftGroupId id,
RaftStorage raftStorage) throws IOException {
- lifeCycle.startAndTransition(() -> {
+ getLifeCycle().startAndTransition(() -> {
super.initialize(server, id, raftStorage);
this.raftGroupId = id;
storage.init(raftStorage);
@@ -304,8 +310,8 @@
@Override
public void pause() {
- lifeCycle.transition(LifeCycle.State.PAUSING);
- lifeCycle.transition(LifeCycle.State.PAUSED);
+ getLifeCycle().transition(LifeCycle.State.PAUSING);
+ getLifeCycle().transition(LifeCycle.State.PAUSED);
ozoneManagerDoubleBuffer.stop();
}
@@ -316,10 +322,14 @@
*/
public void unpause(long newLastAppliedSnaphsotIndex,
long newLastAppliedSnapShotTermIndex) {
- lifeCycle.startAndTransition(() -> {
+ getLifeCycle().startAndTransition(() -> {
this.ozoneManagerDoubleBuffer =
- new OzoneManagerDoubleBuffer(ozoneManager.getMetadataManager(),
- this::updateLastAppliedIndex);
+ new OzoneManagerDoubleBuffer.Builder()
+ .setOmMetadataManager(ozoneManager.getMetadataManager())
+ .setOzoneManagerRatisSnapShot(this::updateLastAppliedIndex)
+ .enableRatis(true)
+ .enableTracing(isTracingEnabled)
+ .build();
handler.updateDoubleBuffer(ozoneManagerDoubleBuffer);
this.setLastAppliedTermIndex(TermIndex.newTermIndex(
newLastAppliedSnapShotTermIndex, newLastAppliedSnaphsotIndex));
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
index 106ba0f..9f19907 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
@@ -34,14 +34,13 @@
import org.apache.hadoop.ozone.om.request.key.OMKeyDeleteRequest;
import org.apache.hadoop.ozone.om.request.key.OMKeyPurgeRequest;
import org.apache.hadoop.ozone.om.request.key.OMKeyRenameRequest;
+import org.apache.hadoop.ozone.om.request.key.OMTrashRecoverRequest;
import org.apache.hadoop.ozone.om.request.key.acl.OMKeyAddAclRequest;
import org.apache.hadoop.ozone.om.request.key.acl.OMKeyRemoveAclRequest;
import org.apache.hadoop.ozone.om.request.key.acl.OMKeySetAclRequest;
import org.apache.hadoop.ozone.om.request.key.acl.prefix.OMPrefixAddAclRequest;
import org.apache.hadoop.ozone.om.request.key.acl.prefix.OMPrefixRemoveAclRequest;
import org.apache.hadoop.ozone.om.request.key.acl.prefix.OMPrefixSetAclRequest;
-import org.apache.hadoop.ozone.om.request.s3.bucket.S3BucketCreateRequest;
-import org.apache.hadoop.ozone.om.request.s3.bucket.S3BucketDeleteRequest;
import org.apache.hadoop.ozone.om.request.s3.multipart.S3InitiateMultipartUploadRequest;
import org.apache.hadoop.ozone.om.request.s3.multipart.S3MultipartUploadAbortRequest;
import org.apache.hadoop.ozone.om.request.s3.multipart.S3MultipartUploadCommitPartRequest;
@@ -122,10 +121,6 @@
return new OMFileCreateRequest(omRequest);
case PurgeKeys:
return new OMKeyPurgeRequest(omRequest);
- case CreateS3Bucket:
- return new S3BucketCreateRequest(omRequest);
- case DeleteS3Bucket:
- return new S3BucketDeleteRequest(omRequest);
case InitiateMultiPartUpload:
return new S3InitiateMultipartUploadRequest(omRequest);
case CommitMultiPartUpload:
@@ -146,6 +141,8 @@
return new OMRenewDelegationTokenRequest(omRequest);
case GetS3Secret:
return new S3GetSecretRequest(omRequest);
+ case RecoverTrash:
+ return new OMTrashRecoverRequest(omRequest);
default:
throw new IllegalStateException("Unrecognized write command " +
"type request" + cmdType);
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java
index 95ead23..c075cd7 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java
@@ -24,11 +24,14 @@
import java.util.stream.Collectors;
import com.google.common.base.Optional;
+
+import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -88,6 +91,8 @@
CreateBucketRequest createBucketRequest =
getOmRequest().getCreateBucketRequest();
BucketInfo bucketInfo = createBucketRequest.getBucketInfo();
+ // Verify resource name
+ OmUtils.validateBucketName(bucketInfo.getBucketName());
// Get KMS provider.
KeyProviderCryptoExtension kmsProvider =
@@ -128,9 +133,8 @@
String volumeName = bucketInfo.getVolumeName();
String bucketName = bucketInfo.getBucketName();
- OMResponse.Builder omResponse = OMResponse.newBuilder().setCmdType(
- OzoneManagerProtocolProtos.Type.CreateBucket).setStatus(
- OzoneManagerProtocolProtos.Status.OK);
+ OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
+ getOmRequest());
OmBucketInfo omBucketInfo = OmBucketInfo.getFromProtobuf(bucketInfo);
AuditLogger auditLogger = ozoneManager.getAuditLogger();
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java
index b84aa71..8548a35 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java
@@ -23,6 +23,7 @@
import com.google.common.base.Optional;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -80,10 +81,8 @@
String bucketName = deleteBucketRequest.getBucketName();
// Generate end user response
- OMResponse.Builder omResponse = OMResponse.newBuilder()
- .setStatus(OzoneManagerProtocolProtos.Status.OK)
- .setCmdType(omRequest.getCmdType());
-
+ OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
+ getOmRequest());
AuditLogger auditLogger = ozoneManager.getAuditLogger();
Map<String, String> auditMap = buildVolumeAuditMap(volumeName);
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java
index ae106f7..2288de7 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java
@@ -25,6 +25,7 @@
import org.apache.hadoop.ozone.audit.AuditLogger;
import org.apache.hadoop.ozone.audit.OMAction;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -89,9 +90,8 @@
String volumeName = bucketArgs.getVolumeName();
String bucketName = bucketArgs.getBucketName();
- OMResponse.Builder omResponse = OMResponse.newBuilder().setCmdType(
- OzoneManagerProtocolProtos.Type.CreateBucket).setStatus(
- OzoneManagerProtocolProtos.Status.OK);
+ OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
+ getOmRequest());
OmBucketInfo omBucketInfo = null;
AuditLogger auditLogger = ozoneManager.getAuditLogger();
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketAddAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketAddAclRequest.java
index 54b7a90..78afeff 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketAddAclRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketAddAclRequest.java
@@ -23,6 +23,7 @@
import com.google.common.collect.Lists;
import org.apache.hadoop.ozone.om.OMMetrics;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.util.BooleanBiFunction;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -78,10 +79,7 @@
@Override
OMResponse.Builder onInit() {
- return OMResponse.newBuilder().setCmdType(
- OzoneManagerProtocolProtos.Type.AddAcl).setStatus(
- OzoneManagerProtocolProtos.Status.OK).setSuccess(true);
-
+ return OmResponseUtil.getOMResponseBuilder(getOmRequest());
}
@Override
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketRemoveAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketRemoveAclRequest.java
index 4921324..5268309 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketRemoveAclRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketRemoveAclRequest.java
@@ -21,6 +21,7 @@
import java.io.IOException;
import java.util.List;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -75,10 +76,7 @@
@Override
OMResponse.Builder onInit() {
- return OMResponse.newBuilder().setCmdType(
- OzoneManagerProtocolProtos.Type.RemoveAcl).setStatus(
- OzoneManagerProtocolProtos.Status.OK).setSuccess(true);
-
+ return OmResponseUtil.getOMResponseBuilder(getOmRequest());
}
@Override
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketSetAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketSetAclRequest.java
index 980c415..66df8c6 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketSetAclRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketSetAclRequest.java
@@ -22,6 +22,7 @@
import java.util.ArrayList;
import java.util.List;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -76,10 +77,7 @@
@Override
OMResponse.Builder onInit() {
- return OMResponse.newBuilder().setCmdType(
- OzoneManagerProtocolProtos.Type.SetAcl).setStatus(
- OzoneManagerProtocolProtos.Status.OK).setSuccess(true);
-
+ return OmResponseUtil.getOMResponseBuilder(getOmRequest());
}
@Override
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java
index b8b426a..7f860fc 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java
@@ -31,18 +31,17 @@
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.om.exceptions.OMReplayException;
import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.ozone.audit.AuditLogger;
import org.apache.hadoop.ozone.audit.OMAction;
@@ -132,11 +131,9 @@
String bucketName = keyArgs.getBucketName();
String keyName = keyArgs.getKeyName();
- OMResponse.Builder omResponse = OMResponse.newBuilder()
- .setCmdType(OzoneManagerProtocolProtos.Type.CreateDirectory)
- .setStatus(OzoneManagerProtocolProtos.Status.OK)
- .setCreateDirectoryResponse(CreateDirectoryResponse.newBuilder());
-
+ OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
+ getOmRequest());
+ omResponse.setCreateDirectoryResponse(CreateDirectoryResponse.newBuilder());
OMMetrics omMetrics = ozoneManager.getMetrics();
omMetrics.incNumCreateDirectory();
@@ -190,7 +187,7 @@
long baseObjId = OMFileRequest.getObjIDFromTxId(trxnLogIndex);
List<OzoneAcl> inheritAcls = omPathInfo.getAcls();
- dirKeyInfo = createDirectoryKeyInfoWithACL(ozoneManager, keyName,
+ dirKeyInfo = createDirectoryKeyInfoWithACL(keyName,
keyArgs, baseObjId,
OzoneAclUtil.fromProtobuf(keyArgs.getAclsList()), trxnLogIndex);
@@ -286,7 +283,7 @@
LOG.debug("missing parent {} getting added to KeyTable", missingKey);
// what about keyArgs for parent directories? TODO
- OmKeyInfo parentKeyInfo = createDirectoryKeyInfoWithACL(ozoneManager,
+ OmKeyInfo parentKeyInfo = createDirectoryKeyInfoWithACL(
missingKey, keyArgs, nextObjId, inheritAcls, trxnLogIndex);
objectCount++;
@@ -345,42 +342,26 @@
* without initializing ACLs from the KeyArgs - used for intermediate
* directories which get created internally/recursively during file
* and directory create.
- * @param ozoneManager
* @param keyName
* @param keyArgs
* @param objectId
* @param transactionIndex
* @return the OmKeyInfo structure
- * @throws IOException
*/
public static OmKeyInfo createDirectoryKeyInfoWithACL(
- OzoneManager ozoneManager, String keyName, KeyArgs keyArgs,
- long objectId, List<OzoneAcl> inheritAcls,
- long transactionIndex) throws IOException {
- String volumeName = keyArgs.getVolumeName();
- String bucketName = keyArgs.getBucketName();
- OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
-
- OmBucketInfo omBucketInfo = omMetadataManager.getBucketTable().get(
- omMetadataManager.getBucketKey(volumeName, bucketName));
- return dirKeyInfoBuilderNoACL(ozoneManager, omBucketInfo, volumeName,
- bucketName, keyName, keyArgs, objectId)
- .setAcls(inheritAcls)
- .setUpdateID(transactionIndex)
- .build();
+ String keyName, KeyArgs keyArgs, long objectId,
+ List<OzoneAcl> inheritAcls, long transactionIndex) {
+ return dirKeyInfoBuilderNoACL(keyName, keyArgs, objectId)
+ .setAcls(inheritAcls).setUpdateID(transactionIndex).build();
}
- private static OmKeyInfo.Builder dirKeyInfoBuilderNoACL(
- OzoneManager ozoneManager, OmBucketInfo omBucketInfo, String volumeName,
- String bucketName, String keyName, KeyArgs keyArgs, long objectId)
- throws IOException {
- Optional<FileEncryptionInfo> encryptionInfo =
- getFileEncryptionInfo(ozoneManager, omBucketInfo);
+ private static OmKeyInfo.Builder dirKeyInfoBuilderNoACL(String keyName,
+ KeyArgs keyArgs, long objectId) {
String dirName = OzoneFSUtils.addTrailingSlashIfNeeded(keyName);
return new OmKeyInfo.Builder()
- .setVolumeName(volumeName)
- .setBucketName(bucketName)
+ .setVolumeName(keyArgs.getVolumeName())
+ .setBucketName(keyArgs.getBucketName())
.setKeyName(dirName)
.setOmKeyLocationInfos(Collections.singletonList(
new OmKeyLocationInfoGroup(0, new ArrayList<>())))
@@ -389,8 +370,8 @@
.setDataSize(0)
.setReplicationType(HddsProtos.ReplicationType.RATIS)
.setReplicationFactor(HddsProtos.ReplicationFactor.ONE)
- .setFileEncryptionInfo(encryptionInfo.orNull())
- .setObjectID(objectId);
+ .setObjectID(objectId)
+ .setUpdateID(objectId);
}
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
index 8bda192..1a31cac 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
@@ -29,11 +29,11 @@
import com.google.common.base.Preconditions;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.om.exceptions.OMReplayException;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.om.response.file.OMFileCreateResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
import org.apache.hadoop.ozone.audit.OMAction;
@@ -58,8 +58,6 @@
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.OMRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
- .Status;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.Type;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
@@ -139,6 +137,7 @@
newKeyArgs.addAllKeyLocations(omKeyLocationInfoList.stream()
.map(OmKeyLocationInfo::getProtobuf).collect(Collectors.toList()));
+ generateRequiredEncryptionInfo(keyArgs, newKeyArgs, ozoneManager);
CreateFileRequest.Builder newCreateFileRequest =
createFileRequest.toBuilder().setKeyArgs(newKeyArgs)
.setClientID(UniqueId.next());
@@ -177,15 +176,14 @@
OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
boolean acquiredLock = false;
- Optional<FileEncryptionInfo> encryptionInfo = Optional.absent();
+
OmKeyInfo omKeyInfo = null;
final List<OmKeyLocationInfo> locations = new ArrayList<>();
List<OmKeyInfo> missingParentInfos;
OMClientResponse omClientResponse = null;
- OMResponse.Builder omResponse = OMResponse.newBuilder()
- .setCmdType(Type.CreateFile)
- .setStatus(Status.OK);
+ OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
+ getOmRequest());
IOException exception = null;
Result result = null;
try {
@@ -257,10 +255,9 @@
// do open key
OmBucketInfo bucketInfo = omMetadataManager.getBucketTable().get(
omMetadataManager.getBucketKey(volumeName, bucketName));
- encryptionInfo = getFileEncryptionInfo(ozoneManager, bucketInfo);
omKeyInfo = prepareKeyInfo(omMetadataManager, keyArgs, dbKeyInfo,
- keyArgs.getDataSize(), locations, encryptionInfo.orNull(),
+ keyArgs.getDataSize(), locations, getFileEncryptionInfo(keyArgs),
ozoneManager.getPrefixManager(), bucketInfo, trxnLogIndex,
ozoneManager.isRatisEnabled());
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
index 3bd305b..348c96a 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
@@ -27,6 +27,7 @@
import com.google.common.base.Preconditions;
import org.apache.hadoop.ozone.om.exceptions.OMReplayException;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
@@ -164,9 +165,8 @@
String openKeyName = omMetadataManager.getOpenKey(volumeName, bucketName,
keyName, clientID);
- OMResponse.Builder omResponse = OMResponse.newBuilder().setCmdType(
- OzoneManagerProtocolProtos.Type.AllocateBlock).setStatus(
- OzoneManagerProtocolProtos.Status.OK).setSuccess(true);
+ OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
+ getOmRequest());
OMClientResponse omClientResponse = null;
OmKeyInfo openKeyInfo = null;
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
index e059ca0..8a99e4f 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
@@ -26,6 +26,7 @@
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -41,15 +42,14 @@
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.om.response.key.OMKeyCommitResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.CommitKeyRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
- .CommitKeyResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.KeyArgs;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.OMRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .OMResponse;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
@@ -111,11 +111,8 @@
Map<String, String> auditMap = buildKeyArgsAuditMap(commitKeyArgs);
- OzoneManagerProtocolProtos.OMResponse.Builder omResponse =
- OzoneManagerProtocolProtos.OMResponse.newBuilder().setCmdType(
- OzoneManagerProtocolProtos.Type.CommitKey).setStatus(
- OzoneManagerProtocolProtos.Status.OK).setSuccess(true)
- .setCommitKeyResponse(CommitKeyResponse.newBuilder());
+ OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
+ getOmRequest());
IOException exception = null;
OmKeyInfo omKeyInfo = null;
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
index 7c3cae3..4b4aeda 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
@@ -26,10 +26,10 @@
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
@@ -56,8 +56,6 @@
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.OMRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
- .Status;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.Type;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
@@ -143,6 +141,7 @@
newKeyArgs = keyArgs.toBuilder().setModificationTime(Time.now());
}
+ generateRequiredEncryptionInfo(keyArgs, newKeyArgs, ozoneManager);
newCreateKeyRequest =
createKeyRequest.toBuilder().setKeyArgs(newKeyArgs)
.setClientID(UniqueId.next());
@@ -169,12 +168,11 @@
OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
OmKeyInfo omKeyInfo = null;
final List< OmKeyLocationInfo > locations = new ArrayList<>();
- Optional<FileEncryptionInfo> encryptionInfo = Optional.absent();
+
boolean acquireLock = false;
OMClientResponse omClientResponse = null;
- OMResponse.Builder omResponse = OMResponse.newBuilder()
- .setCmdType(Type.CreateKey)
- .setStatus(Status.OK);
+ OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
+ getOmRequest());
IOException exception = null;
Result result = null;
try {
@@ -213,10 +211,9 @@
OmBucketInfo bucketInfo = omMetadataManager.getBucketTable().get(
omMetadataManager.getBucketKey(volumeName, bucketName));
- encryptionInfo = getFileEncryptionInfo(ozoneManager, bucketInfo);
omKeyInfo = prepareKeyInfo(omMetadataManager, keyArgs, dbKeyInfo,
- keyArgs.getDataSize(), locations, encryptionInfo.orNull(),
+ keyArgs.getDataSize(), locations, getFileEncryptionInfo(keyArgs),
ozoneManager.getPrefixManager(), bucketInfo, trxnLogIndex,
ozoneManager.isRatisEnabled());
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java
index f0e77cb..1f1b0fb 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java
@@ -23,6 +23,7 @@
import com.google.common.base.Optional;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.slf4j.Logger;
@@ -48,10 +49,6 @@
.OMRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.OMResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
- .Status;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
- .Type;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
@@ -107,10 +104,8 @@
Map<String, String> auditMap = buildKeyArgsAuditMap(deleteKeyArgs);
- OMResponse.Builder omResponse = OMResponse.newBuilder()
- .setCmdType(Type.DeleteKey)
- .setStatus(Status.OK)
- .setSuccess(true);
+ OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
+ getOmRequest());
OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
IOException exception = null;
boolean acquiredLock = false;
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java
index 3ea904a..f7783db 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java
@@ -25,15 +25,13 @@
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.om.response.key.OMKeyPurgeResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeletedKeys;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PurgeKeysRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PurgeKeysResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -64,11 +62,8 @@
.getDeletedKeysList();
List<String> keysToBePurgedList = new ArrayList<>();
- OMResponse.Builder omResponse = OMResponse.newBuilder()
- .setCmdType(Type.PurgeKeys)
- .setPurgeKeysResponse(PurgeKeysResponse.newBuilder().build())
- .setStatus(Status.OK)
- .setSuccess(true);
+ OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
+ getOmRequest());
OMClientResponse omClientResponse = null;
boolean success = true;
IOException exception = null;
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java
index ca97067..d6fd884 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java
@@ -25,6 +25,7 @@
import com.google.common.base.Preconditions;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.slf4j.Logger;
@@ -43,6 +44,7 @@
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.KeyArgs;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.RenameKeyRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
@@ -115,10 +117,8 @@
Map<String, String> auditMap =
buildAuditMap(renameKeyArgs, renameKeyRequest);
- OzoneManagerProtocolProtos.OMResponse.Builder omResponse =
- OzoneManagerProtocolProtos.OMResponse.newBuilder().setCmdType(
- OzoneManagerProtocolProtos.Type.CommitKey).setStatus(
- OzoneManagerProtocolProtos.Status.OK).setSuccess(true);
+ OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
+ getOmRequest());
OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
boolean acquiredLock = false;
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
index d6fd048..0aec04d 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
@@ -41,6 +41,7 @@
import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo;
import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil;
import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
+import org.apache.hadoop.ozone.protocolPB.OMPBHelper;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.slf4j.Logger;
@@ -48,7 +49,6 @@
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension
.EncryptedKeyVersion;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -74,6 +74,7 @@
.BUCKET_NOT_FOUND;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
.VOLUME_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK;
import static org.apache.hadoop.util.Time.monotonicNow;
/**
@@ -187,12 +188,6 @@
Optional<FileEncryptionInfo> encInfo = Optional.absent();
BucketEncryptionKeyInfo ezInfo = bucketInfo.getEncryptionKeyInfo();
if (ezInfo != null) {
- if (ozoneManager.getKmsProvider() == null) {
- throw new OMException("Invalid KMS provider, check configuration " +
- CommonConfigurationKeys.HADOOP_SECURITY_KEY_PROVIDER_PATH,
- OMException.ResultCodes.INVALID_KMS_PROVIDER);
- }
-
final String ezKeyName = ezInfo.getKeyName();
EncryptedKeyVersion edek = generateEDEK(ozoneManager, ezKeyName);
encInfo = Optional.of(new FileEncryptionInfo(ezInfo.getSuite(),
@@ -451,4 +446,75 @@
checkKeyAcls(ozoneManager, volume, bucket, keyNameForAclCheck,
aclType, OzoneObj.ResourceType.KEY);
}
+
+ /**
+ * Generate EncryptionInfo and set in to newKeyArgs.
+ * @param keyArgs
+ * @param newKeyArgs
+ * @param ozoneManager
+ */
+ protected void generateRequiredEncryptionInfo(KeyArgs keyArgs,
+ KeyArgs.Builder newKeyArgs, OzoneManager ozoneManager)
+ throws IOException {
+
+ String volumeName = keyArgs.getVolumeName();
+ String bucketName = keyArgs.getBucketName();
+
+ boolean acquireLock = false;
+ OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
+
+ // When TDE is enabled, we are doing a DB read in pre-execute. As for
+ // most of the operations we don't read from DB because of our isLeader
+ // semantics. This issue will be solved with implementation of leader
+ // leases which provider strong leader semantics in the system.
+
+ // If KMS is not enabled, follow the normal approach of execution of not
+ // reading DB in pre-execute.
+ if (ozoneManager.getKmsProvider() != null) {
+ try {
+ acquireLock = omMetadataManager.getLock().acquireReadLock(
+ BUCKET_LOCK, volumeName, bucketName);
+
+
+ OmBucketInfo bucketInfo = omMetadataManager.getBucketTable().get(
+ omMetadataManager.getBucketKey(volumeName, bucketName));
+
+
+ // Don't throw exception of bucket not found when bucketinfo is not
+ // null. If bucketinfo is null, later when request
+ // is submitted and if bucket does not really exist it will fail in
+ // applyTransaction step. Why we are doing this is if OM thinks it is
+ // the leader, but it is not, we don't want to fail request in this
+ // case. As anyway when it submits request to ratis it will fail with
+ // not leader exception, and client will retry on correct leader and
+ // request will be executed.
+ if (bucketInfo != null) {
+ Optional< FileEncryptionInfo > encryptionInfo =
+ getFileEncryptionInfo(ozoneManager, bucketInfo);
+ if (encryptionInfo.isPresent()) {
+ newKeyArgs.setFileEncryptionInfo(
+ OMPBHelper.convert(encryptionInfo.get()));
+ }
+ }
+ } finally {
+ if (acquireLock) {
+ omMetadataManager.getLock().releaseReadLock(
+ BUCKET_LOCK, volumeName, bucketName);
+ }
+ }
+ }
+ }
+
+ /**
+ * Get FileEncryptionInfoProto from KeyArgs.
+ * @param keyArgs
+ * @return
+ */
+ protected FileEncryptionInfo getFileEncryptionInfo(KeyArgs keyArgs) {
+ FileEncryptionInfo encryptionInfo = null;
+ if (keyArgs.hasFileEncryptionInfo()) {
+ encryptionInfo = OMPBHelper.convert(keyArgs.getFileEncryptionInfo());
+ }
+ return encryptionInfo;
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMTrashRecoverRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMTrashRecoverRequest.java
new file mode 100644
index 0000000..eac7842
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMTrashRecoverRequest.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.request.key;
+
+import java.io.IOException;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
+import org.apache.hadoop.ozone.om.response.key.OMTrashRecoverResponse;
+import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .RecoverTrashRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .OMResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .OMRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status;
+
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK;
+
+/**
+ * Handles RecoverTrash request.
+ */
+public class OMTrashRecoverRequest extends OMKeyRequest {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(OMTrashRecoverRequest.class);
+
+ public OMTrashRecoverRequest(OMRequest omRequest) {
+ super(omRequest);
+ }
+
+ @Override
+ public OMRequest preExecute(OzoneManager ozoneManager) {
+ RecoverTrashRequest recoverTrashRequest = getOmRequest()
+ .getRecoverTrashRequest();
+ Preconditions.checkNotNull(recoverTrashRequest);
+
+ return getOmRequest().toBuilder().build();
+ }
+
+ @Override
+ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
+ long transactionLogIndex,
+ OzoneManagerDoubleBufferHelper ozoneManagerDoubleBufferHelper) {
+ RecoverTrashRequest recoverTrashRequest = getOmRequest()
+ .getRecoverTrashRequest();
+ Preconditions.checkNotNull(recoverTrashRequest);
+
+ String volumeName = recoverTrashRequest.getVolumeName();
+ String bucketName = recoverTrashRequest.getBucketName();
+ String keyName = recoverTrashRequest.getKeyName();
+ String destinationBucket = recoverTrashRequest.getDestinationBucket();
+
+ /** TODO: HDDS-2818. New Metrics for Trash Key Recover and Fails.
+ * OMMetrics omMetrics = ozoneManager.getMetrics();
+ */
+
+ OMResponse.Builder omResponse = OMResponse.newBuilder()
+ .setCmdType(Type.RecoverTrash).setStatus(Status.OK)
+ .setSuccess(true);
+
+ OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
+ boolean acquireLock = false;
+ OMClientResponse omClientResponse = null;
+ try {
+ // Check acl for the destination bucket.
+ checkBucketAcls(ozoneManager, volumeName, destinationBucket, keyName,
+ IAccessAuthorizer.ACLType.WRITE);
+
+ acquireLock = omMetadataManager.getLock()
+ .acquireWriteLock(BUCKET_LOCK, volumeName, destinationBucket);
+
+ // Validate.
+ validateBucketAndVolume(omMetadataManager, volumeName, bucketName);
+ validateBucketAndVolume(omMetadataManager, volumeName, destinationBucket);
+
+
+ /** TODO: HDDS-2425. HDDS-2426.
+ * Update cache.
+ * omMetadataManager.getKeyTable().addCacheEntry(
+ * new CacheKey<>(),
+ * new CacheValue<>()
+ * );
+ *
+ * Execute recovering trash in non-existing bucket.
+ * Execute recovering trash in existing bucket.
+ * omClientResponse = new OMTrashRecoverResponse(omKeyInfo,
+ * omResponse.setRecoverTrashResponse(
+ * RecoverTrashResponse.newBuilder())
+ * .build());
+ */
+ omClientResponse = null;
+
+ } catch (IOException ex) {
+ LOG.error("Fail for recovering trash.", ex);
+ omClientResponse = new OMTrashRecoverResponse(null,
+ createErrorOMResponse(omResponse, ex));
+ } finally {
+ if (omClientResponse != null) {
+ omClientResponse.setFlushFuture(
+ ozoneManagerDoubleBufferHelper.add(omClientResponse,
+ transactionLogIndex));
+ }
+ if (acquireLock) {
+ omMetadataManager.getLock().releaseWriteLock(BUCKET_LOCK, volumeName,
+ destinationBucket);
+ }
+ }
+
+ return omClientResponse;
+ }
+
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java
index d0e0043..444c0df 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java
@@ -24,6 +24,7 @@
import com.google.common.collect.Lists;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.om.response.key.acl.OMKeyAclResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.slf4j.Logger;
@@ -61,10 +62,7 @@
@Override
OMResponse.Builder onInit() {
- return OMResponse.newBuilder().setCmdType(
- OzoneManagerProtocolProtos.Type.AddAcl).setStatus(
- OzoneManagerProtocolProtos.Status.OK).setSuccess(true);
-
+ return OmResponseUtil.getOMResponseBuilder(getOmRequest());
}
@Override
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java
index 9032904..18e999d 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java
@@ -24,6 +24,7 @@
import com.google.common.collect.Lists;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.om.response.key.acl.OMKeyAclResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.slf4j.Logger;
@@ -61,10 +62,7 @@
@Override
OMResponse.Builder onInit() {
- return OMResponse.newBuilder().setCmdType(
- OzoneManagerProtocolProtos.Type.RemoveAcl).setStatus(
- OzoneManagerProtocolProtos.Status.OK).setSuccess(true);
-
+ return OmResponseUtil.getOMResponseBuilder(getOmRequest());
}
@Override
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java
index ba4e6ff..d8dbe77 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java
@@ -25,6 +25,7 @@
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.om.response.key.acl.OMKeyAclResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.slf4j.Logger;
@@ -62,10 +63,7 @@
@Override
OMResponse.Builder onInit() {
- return OMResponse.newBuilder().setCmdType(
- OzoneManagerProtocolProtos.Type.SetAcl).setStatus(
- OzoneManagerProtocolProtos.Status.OK).setSuccess(true);
-
+ return OmResponseUtil.getOMResponseBuilder(getOmRequest());
}
@Override
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/prefix/OMPrefixAddAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/prefix/OMPrefixAddAclRequest.java
index c8830c8..bd25e07 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/prefix/OMPrefixAddAclRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/prefix/OMPrefixAddAclRequest.java
@@ -26,6 +26,7 @@
import org.apache.hadoop.ozone.om.PrefixManagerImpl;
import org.apache.hadoop.ozone.om.PrefixManagerImpl.OMPrefixAclOpResult;
import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.om.response.key.acl.prefix.OMPrefixAclResponse;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
@@ -71,10 +72,7 @@
@Override
OMResponse.Builder onInit() {
- return OMResponse.newBuilder().setCmdType(
- OzoneManagerProtocolProtos.Type.AddAcl).setStatus(
- OzoneManagerProtocolProtos.Status.OK).setSuccess(true);
-
+ return OmResponseUtil.getOMResponseBuilder(getOmRequest());
}
@Override
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/prefix/OMPrefixRemoveAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/prefix/OMPrefixRemoveAclRequest.java
index 7300834..72c199c 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/prefix/OMPrefixRemoveAclRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/prefix/OMPrefixRemoveAclRequest.java
@@ -26,6 +26,7 @@
import org.apache.hadoop.ozone.om.PrefixManagerImpl;
import org.apache.hadoop.ozone.om.PrefixManagerImpl.OMPrefixAclOpResult;
import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.om.response.key.acl.prefix.OMPrefixAclResponse;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
@@ -68,10 +69,7 @@
@Override
OMResponse.Builder onInit() {
- return OMResponse.newBuilder().setCmdType(
- OzoneManagerProtocolProtos.Type.RemoveAcl).setStatus(
- OzoneManagerProtocolProtos.Status.OK).setSuccess(true);
-
+ return OmResponseUtil.getOMResponseBuilder(getOmRequest());
}
@Override
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/prefix/OMPrefixSetAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/prefix/OMPrefixSetAclRequest.java
index c8ada2c..122ada1 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/prefix/OMPrefixSetAclRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/prefix/OMPrefixSetAclRequest.java
@@ -26,6 +26,7 @@
import org.apache.hadoop.ozone.om.PrefixManagerImpl;
import org.apache.hadoop.ozone.om.PrefixManagerImpl.OMPrefixAclOpResult;
import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.om.response.key.acl.prefix.OMPrefixAclResponse;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
@@ -69,10 +70,7 @@
@Override
OMResponse.Builder onInit() {
- return OMResponse.newBuilder().setCmdType(
- OzoneManagerProtocolProtos.Type.SetAcl).setStatus(
- OzoneManagerProtocolProtos.Status.OK).setSuccess(true);
-
+ return OmResponseUtil.getOMResponseBuilder(getOmRequest());
}
@Override
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/bucket/S3BucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/bucket/S3BucketCreateRequest.java
deleted file mode 100644
index 9f916ef..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/bucket/S3BucketCreateRequest.java
+++ /dev/null
@@ -1,412 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.om.request.s3.bucket;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
-import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.hdds.protocol.StorageType;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.audit.OMAction;
-import org.apache.hadoop.ozone.om.OMMetadataManager;
-import org.apache.hadoop.ozone.om.OMMetrics;
-import org.apache.hadoop.ozone.om.OzoneManager;
-import org.apache.hadoop.ozone.om.exceptions.OMException;
-import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
-import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
-import org.apache.hadoop.ozone.om.request.volume.OMVolumeRequest;
-import org.apache.hadoop.ozone.om.response.OMClientResponse;
-import org.apache.hadoop.ozone.om.response.bucket.OMBucketCreateResponse;
-import org.apache.hadoop.ozone.om.response.s3.bucket.S3BucketCreateResponse;
-import org.apache.hadoop.ozone.om.response.volume.OMVolumeCreateResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
- .OMRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
- .OMResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
- .S3CreateBucketRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
- .S3CreateBucketResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
- .S3CreateVolumeInfo;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.UserVolumeInfo;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
-import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
-
-import static org.apache.hadoop.ozone.OzoneConsts.OM_S3_VOLUME_PREFIX;
-import static org.apache.hadoop.ozone.OzoneConsts.S3_BUCKET_MAX_LENGTH;
-import static org.apache.hadoop.ozone.OzoneConsts.S3_BUCKET_MIN_LENGTH;
-import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK;
-import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.S3_BUCKET_LOCK;
-import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.USER_LOCK;
-import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.VOLUME_LOCK;
-
-/**
- * Handles S3 Bucket create request.
- */
-public class S3BucketCreateRequest extends OMVolumeRequest {
-
- private static final String S3_ADMIN_NAME = "OzoneS3Manager";
-
- private static final Logger LOG =
- LoggerFactory.getLogger(S3CreateBucketRequest.class);
-
- public S3BucketCreateRequest(OMRequest omRequest) {
- super(omRequest);
- }
-
- @Override
- public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
- S3CreateBucketRequest s3CreateBucketRequest =
- getOmRequest().getCreateS3BucketRequest();
- Preconditions.checkNotNull(s3CreateBucketRequest);
-
- S3CreateBucketRequest.Builder newS3CreateBucketRequest =
- s3CreateBucketRequest.toBuilder().setS3CreateVolumeInfo(
- S3CreateVolumeInfo.newBuilder().setCreationTime(Time.now()));
-
- // TODO: Do we need to enforce the bucket rules in this code path?
- // https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html
-
- // For now only checked the length.
- int bucketLength = s3CreateBucketRequest.getS3Bucketname().length();
- if (bucketLength < S3_BUCKET_MIN_LENGTH ||
- bucketLength >= S3_BUCKET_MAX_LENGTH) {
- throw new OMException("S3BucketName must be at least 3 and not more " +
- "than 63 characters long",
- OMException.ResultCodes.S3_BUCKET_INVALID_LENGTH);
- }
-
- return getOmRequest().toBuilder()
- .setCreateS3BucketRequest(newS3CreateBucketRequest)
- .setUserInfo(getUserInfo()).build();
- }
-
- @Override
- public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
- long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) {
-
- S3CreateBucketRequest s3CreateBucketRequest = getOmRequest()
- .getCreateS3BucketRequest();
- String userName = s3CreateBucketRequest.getUserName();
- String s3BucketName = s3CreateBucketRequest.getS3Bucketname();
-
- OMResponse.Builder omResponse = OMResponse.newBuilder().setCmdType(
- OzoneManagerProtocolProtos.Type.CreateS3Bucket).setStatus(
- OzoneManagerProtocolProtos.Status.OK).setSuccess(true);
-
- OMMetrics omMetrics = ozoneManager.getMetrics();
- omMetrics.incNumS3BucketCreates();
-
- // When s3 Bucket is created, we internally create ozone volume/ozone
- // bucket. Ozone volume name is generated from userName by calling
- // formatOzoneVolumeName. Ozone bucket name is same as s3 bucket name.
- // In S3 buckets are unique, so we create a mapping like s3BucketName ->
- // ozoneVolume/ozoneBucket and add it to s3 mapping table. If
- // s3BucketName exists in mapping table, bucket already exist or we go
- // ahead and create a bucket.
- OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
- IOException exception = null;
-
- boolean volumeCreated = false, acquiredVolumeLock = false,
- acquiredUserLock = false, acquiredS3Lock = false, success = true;
- String volumeName = formatOzoneVolumeName(userName);
- OMClientResponse omClientResponse = null;
-
- try {
- // TODO to support S3 ACL later.
- acquiredS3Lock = omMetadataManager.getLock().acquireWriteLock(
- S3_BUCKET_LOCK, s3BucketName);
-
- // First check if this s3Bucket exists in S3 table
- if (omMetadataManager.getS3Table().isExist(s3BucketName)) {
- // Check if Bucket exists in Bucket Table
- String omBucketKey = omMetadataManager.getBucketKey(volumeName,
- s3BucketName);
- OmBucketInfo dbBucketInfo = omMetadataManager.getBucketTable()
- .get(omBucketKey);
- if (dbBucketInfo != null) {
- // Check if this transaction is a replay of ratis logs.
- if (isReplay(ozoneManager, dbBucketInfo, trxnLogIndex)) {
- // Replay implies the response has already been returned to
- // the client. So take no further action and return a dummy
- // OMClientResponse.
- LOG.debug("Replayed Transaction {} ignored. Request: {}",
- trxnLogIndex, s3CreateBucketRequest);
- return new S3BucketCreateResponse(
- createReplayOMResponse(omResponse));
- }
- } else {
- throw new OMException("S3Bucket " + s3BucketName + " mapping " +
- "already exists in S3 table but Bucket does not exist.",
- OMException.ResultCodes.S3_BUCKET_ALREADY_EXISTS);
- }
- throw new OMException("S3Bucket " + s3BucketName + " already exists",
- OMException.ResultCodes.S3_BUCKET_ALREADY_EXISTS);
- }
-
- OMVolumeCreateResponse omVolumeCreateResponse = null;
- try {
- acquiredVolumeLock = omMetadataManager.getLock().acquireWriteLock(
- VOLUME_LOCK, volumeName);
- acquiredUserLock = omMetadataManager.getLock().acquireWriteLock(
- USER_LOCK, userName);
- // Check if volume exists, if it does not exist create ozone volume.
- String volumeKey = omMetadataManager.getVolumeKey(volumeName);
- if (!omMetadataManager.getVolumeTable().isExist(volumeKey)) {
- // A replay transaction can reach here only if the volume has been
- // deleted in later transactions. Hence, we can continue with this
- // request irrespective of whether it is a replay or not.
- OmVolumeArgs omVolumeArgs = createOmVolumeArgs(volumeName, userName,
- s3CreateBucketRequest.getS3CreateVolumeInfo().getCreationTime(),
- trxnLogIndex);
- UserVolumeInfo volumeList = omMetadataManager.getUserTable().get(
- omMetadataManager.getUserKey(userName));
- volumeList = addVolumeToOwnerList(volumeList, volumeName, userName,
- ozoneManager.getMaxUserVolumeCount(), trxnLogIndex);
- createVolume(omMetadataManager, omVolumeArgs, volumeList, volumeKey,
- omMetadataManager.getUserKey(userName), trxnLogIndex);
- volumeCreated = true;
- omVolumeCreateResponse = new OMVolumeCreateResponse(
- omResponse.build(), omVolumeArgs, volumeList);
- }
- } finally {
- if (acquiredUserLock) {
- omMetadataManager.getLock().releaseWriteLock(USER_LOCK, userName);
- }
- if (acquiredVolumeLock) {
- omMetadataManager.getLock().releaseWriteLock(VOLUME_LOCK, volumeName);
- }
- }
-
- // check if ozone bucket exists, if not create ozone bucket
- OmBucketInfo omBucketInfo = createBucket(ozoneManager, volumeName,
- s3BucketName, userName, s3CreateBucketRequest.getS3CreateVolumeInfo()
- .getCreationTime(), trxnLogIndex);
-
- // Now finally add it to s3 table cache.
- omMetadataManager.getS3Table().addCacheEntry(
- new CacheKey<>(s3BucketName), new CacheValue<>(
- Optional.of(formatS3MappingName(volumeName, s3BucketName)),
- trxnLogIndex));
-
- OMBucketCreateResponse omBucketCreateResponse =
- new OMBucketCreateResponse(omResponse.build(), omBucketInfo);
-
- omClientResponse = new S3BucketCreateResponse(
- omResponse.setCreateS3BucketResponse(
- S3CreateBucketResponse.newBuilder()).build(),
- omVolumeCreateResponse, omBucketCreateResponse, s3BucketName,
- formatS3MappingName(volumeName, s3BucketName));
- } catch (IOException ex) {
- success = false;
- exception = ex;
- omClientResponse = new S3BucketCreateResponse(
- createErrorOMResponse(omResponse, exception));
- } finally {
- addResponseToDoubleBuffer(trxnLogIndex, omClientResponse,
- omDoubleBufferHelper);
- if (acquiredS3Lock) {
- omMetadataManager.getLock().releaseWriteLock(
- S3_BUCKET_LOCK, s3BucketName);
- }
- }
-
- // Performing audit logging outside of the lock.
- auditLog(ozoneManager.getAuditLogger(), buildAuditMessage(
- OMAction.CREATE_S3_BUCKET, buildAuditMap(userName, s3BucketName),
- exception, getOmRequest().getUserInfo()));
-
- if (success) {
- LOG.debug("S3Bucket is successfully created for userName: {}, " +
- "s3BucketName {}, volumeName {}", userName, s3BucketName, volumeName);
- updateMetrics(omMetrics, volumeCreated);
- } else {
- LOG.error("S3Bucket Creation Failed for userName: {}, s3BucketName {}, " +
- "VolumeName {}", userName, s3BucketName, volumeName);
- omMetrics.incNumS3BucketCreateFails();
- }
- return omClientResponse;
- }
-
- private OmBucketInfo createBucket(OzoneManager ozoneManager,
- String volumeName, String s3BucketName, String userName,
- long creationTime, long transactionLogIndex) throws IOException {
- OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
- // check if ozone bucket exists, if it does not exist create ozone
- // bucket
- boolean acquireBucketLock = false;
- OmBucketInfo omBucketInfo = null;
- try {
- acquireBucketLock =
- omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK, volumeName,
- s3BucketName);
- String bucketKey = omMetadataManager.getBucketKey(volumeName,
- s3BucketName);
- if (!omMetadataManager.getBucketTable().isExist(bucketKey)) {
- omBucketInfo = createOmBucketInfo(volumeName, s3BucketName, userName,
- creationTime, transactionLogIndex);
- // Add to bucket table cache.
- omMetadataManager.getBucketTable().addCacheEntry(
- new CacheKey<>(bucketKey),
- new CacheValue<>(Optional.of(omBucketInfo), transactionLogIndex));
- } else {
- // This can happen when a ozone bucket exists already in the
- // volume, but this is not a s3 bucket.
- throw new OMException("Bucket " + s3BucketName + " already exists",
- OMException.ResultCodes.BUCKET_ALREADY_EXISTS);
- }
- } finally {
- if (acquireBucketLock) {
- omMetadataManager.getLock().releaseWriteLock(BUCKET_LOCK, volumeName,
- s3BucketName);
- }
- }
- return omBucketInfo;
- }
-
- /**
- * Increment OMMetrics on success.
- */
- private static void updateMetrics(OMMetrics omMetrics,
- boolean isVolumeCreated) {
- if (isVolumeCreated) {
- omMetrics.incNumVolumes();
- }
- omMetrics.incNumBuckets();
- omMetrics.incNumS3Buckets();
- }
-
- /**
- * Generate Ozone volume name from userName.
- * @param userName
- * @return volume name
- */
- @VisibleForTesting
- public static String formatOzoneVolumeName(String userName) {
- return String.format(OM_S3_VOLUME_PREFIX + "%s", userName);
- }
-
- /**
- * Generate S3Mapping for provided volume and bucket. This information will
- * be persisted in s3 table in OM DB.
- * @param volumeName
- * @param bucketName
- * @return s3Mapping
- */
- @VisibleForTesting
- public static String formatS3MappingName(String volumeName,
- String bucketName) {
- return String.format("%s" + OzoneConsts.OM_KEY_PREFIX + "%s", volumeName,
- bucketName);
- }
-
- /**
- * Create {@link OmVolumeArgs} which needs to be persisted in volume table
- * in OM DB.
- * @param volumeName
- * @param userName
- * @param creationTime
- * @return {@link OmVolumeArgs}
- */
- private OmVolumeArgs createOmVolumeArgs(String volumeName, String userName,
- long creationTime, long transactionLogIndex) throws IOException {
- long objectID = OMFileRequest.getObjIDFromTxId(transactionLogIndex);
- OmVolumeArgs.Builder builder = OmVolumeArgs.newBuilder()
- .setAdminName(S3_ADMIN_NAME).setVolume(volumeName)
- .setQuotaInBytes(OzoneConsts.MAX_QUOTA_IN_BYTES)
- .setOwnerName(userName)
- .setCreationTime(creationTime)
- .setObjectID(objectID)
- .setUpdateID(transactionLogIndex);
-
- // Set default acls.
- for (OzoneAcl acl : getDefaultAcls(userName)) {
- builder.addOzoneAcls(OzoneAcl.toProtobuf(acl));
- }
-
- return builder.build();
- }
-
- /**
- * Create {@link OmBucketInfo} which needs to be persisted in to bucket table
- * in OM DB.
- * @param volumeName
- * @param s3BucketName
- * @param creationTime
- * @return {@link OmBucketInfo}
- */
- private OmBucketInfo createOmBucketInfo(String volumeName,
- String s3BucketName, String userName, long creationTime,
- long transactionLogIndex) {
- //TODO: Now S3Bucket API takes only bucketName as param. In future if we
- // support some configurable options we need to fix this.
- OmBucketInfo.Builder builder = OmBucketInfo.newBuilder()
- .setVolumeName(volumeName)
- .setBucketName(s3BucketName)
- .setIsVersionEnabled(Boolean.FALSE)
- .setStorageType(StorageType.DEFAULT)
- .setCreationTime(creationTime)
- .setObjectID(transactionLogIndex)
- .setUpdateID(transactionLogIndex);
-
- // Set default acls.
- builder.setAcls(getDefaultAcls(userName));
-
- return builder.build();
- }
-
- /**
- * Build auditMap.
- * @param userName
- * @param s3BucketName
- * @return auditMap
- */
- private Map<String, String> buildAuditMap(String userName,
- String s3BucketName) {
- Map<String, String> auditMap = new HashMap<>();
- auditMap.put(userName, OzoneConsts.USERNAME);
- auditMap.put(s3BucketName, OzoneConsts.S3_BUCKET);
- return auditMap;
- }
-
- /**
- * Get default acls.
- * */
- private List<OzoneAcl> getDefaultAcls(String userName) {
- UserGroupInformation ugi = createUGI();
- return OzoneAcl.parseAcls("user:" + (ugi == null ? userName :
- ugi.getUserName()) + ":a,user:" + S3_ADMIN_NAME + ":a");
- }
-}
-
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/bucket/S3BucketDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/bucket/S3BucketDeleteRequest.java
deleted file mode 100644
index 9bf8dc5..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/bucket/S3BucketDeleteRequest.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.om.request.s3.bucket;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import com.google.common.base.Optional;
-import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
-import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.audit.OMAction;
-import org.apache.hadoop.ozone.om.OMMetadataManager;
-import org.apache.hadoop.ozone.om.OMMetrics;
-import org.apache.hadoop.ozone.om.OzoneManager;
-import org.apache.hadoop.ozone.om.exceptions.OMException;
-import org.apache.hadoop.ozone.om.request.volume.OMVolumeRequest;
-import org.apache.hadoop.ozone.om.response.OMClientResponse;
-import org.apache.hadoop.ozone.om.response.s3.bucket.S3BucketDeleteResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
- .OMRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
- .OMResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
- .S3DeleteBucketRequest;
-import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
-import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
-
-import static org.apache.hadoop.ozone.OzoneConsts.S3_BUCKET_MAX_LENGTH;
-import static org.apache.hadoop.ozone.OzoneConsts.S3_BUCKET_MIN_LENGTH;
-import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK;
-import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.S3_BUCKET_LOCK;
-
-/**
- * Handle Create S3Bucket request.
- */
-public class S3BucketDeleteRequest extends OMVolumeRequest {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(S3BucketDeleteRequest.class);
-
- public S3BucketDeleteRequest(OMRequest omRequest) {
- super(omRequest);
- }
-
- public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
- S3DeleteBucketRequest s3DeleteBucketRequest =
- getOmRequest().getDeleteS3BucketRequest();
-
- // TODO: Do we need to enforce the bucket rules in this code path?
- // https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html
-
- // For now only checked the length.
- int bucketLength = s3DeleteBucketRequest.getS3BucketName().length();
- if (bucketLength < S3_BUCKET_MIN_LENGTH ||
- bucketLength >= S3_BUCKET_MAX_LENGTH) {
- throw new OMException("S3BucketName must be at least 3 and not more " +
- "than 63 characters long",
- OMException.ResultCodes.S3_BUCKET_INVALID_LENGTH);
- }
-
- return getOmRequest().toBuilder().setUserInfo(getUserInfo()).build();
-
- }
-
- @Override
- public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
- long transactionLogIndex,
- OzoneManagerDoubleBufferHelper ozoneManagerDoubleBufferHelper) {
- S3DeleteBucketRequest s3DeleteBucketRequest =
- getOmRequest().getDeleteS3BucketRequest();
-
- String s3BucketName = s3DeleteBucketRequest.getS3BucketName();
-
- OMResponse.Builder omResponse = OMResponse.newBuilder().setCmdType(
- OzoneManagerProtocolProtos.Type.DeleteS3Bucket).setStatus(
- OzoneManagerProtocolProtos.Status.OK).setSuccess(true);
-
- OMMetrics omMetrics = ozoneManager.getMetrics();
- omMetrics.incNumS3BucketDeletes();
- IOException exception = null;
- boolean acquiredS3Lock = false;
- boolean acquiredBucketLock = false;
- String volumeName = null;
- OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
- OMClientResponse omClientResponse = null;
- try {
- // TODO to support S3 ACL later.
- acquiredS3Lock = omMetadataManager.getLock().acquireWriteLock(
- S3_BUCKET_LOCK, s3BucketName);
-
- String s3Mapping = omMetadataManager.getS3Table().get(s3BucketName);
-
- if (s3Mapping == null) {
- throw new OMException("S3Bucket " + s3BucketName + " not found",
- OMException.ResultCodes.S3_BUCKET_NOT_FOUND);
- } else {
- volumeName = getOzoneVolumeName(s3Mapping);
-
- acquiredBucketLock =
- omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK,
- volumeName, s3BucketName);
-
- String bucketKey = omMetadataManager.getBucketKey(volumeName,
- s3BucketName);
-
- // Check if Bucket exists in DB.
- OmBucketInfo dbBucketInfo =
- omMetadataManager.getBucketTable().get(bucketKey);
- if (dbBucketInfo != null) {
- // Check if this transaction is a replay of ratis logs.
- // If this is a replay, then the response has already been returned to
- // the client. So take no further action and return a dummy
- // OMClientResponse.
- if (isReplay(ozoneManager, dbBucketInfo, transactionLogIndex)) {
- LOG.debug("Replayed Transaction {} ignored. Request: {}",
- transactionLogIndex, s3DeleteBucketRequest);
- return new S3BucketDeleteResponse(
- createReplayOMResponse(omResponse));
- }
-
- // Update bucket table cache and s3 table cache.
- omMetadataManager.getBucketTable().addCacheEntry(
- new CacheKey<>(bucketKey),
- new CacheValue<>(Optional.absent(), transactionLogIndex));
- omMetadataManager.getS3Table().addCacheEntry(
- new CacheKey<>(s3BucketName),
- new CacheValue<>(Optional.absent(), transactionLogIndex));
- } else {
- LOG.debug("bucket: {} not found ", bucketKey);
- throw new OMException("Bucket doesn't exist",
- OMException.ResultCodes.BUCKET_NOT_FOUND);
- }
- }
-
- omResponse.setDeleteS3BucketResponse(
- OzoneManagerProtocolProtos.S3DeleteBucketResponse.newBuilder());
-
- omClientResponse = new S3BucketDeleteResponse(omResponse.build(),
- s3BucketName, volumeName);
- } catch (IOException ex) {
- exception = ex;
- omClientResponse = new S3BucketDeleteResponse(
- createErrorOMResponse(omResponse, exception));
- } finally {
- addResponseToDoubleBuffer(transactionLogIndex, omClientResponse,
- ozoneManagerDoubleBufferHelper);
- if (acquiredBucketLock) {
- omMetadataManager.getLock().releaseWriteLock(BUCKET_LOCK, volumeName,
- s3BucketName);
- }
- if (acquiredS3Lock) {
- omMetadataManager.getLock().releaseWriteLock(S3_BUCKET_LOCK,
- s3BucketName);
- }
- }
-
- // Performing audit logging outside of the lock.
- auditLog(ozoneManager.getAuditLogger(),
- buildAuditMessage(OMAction.DELETE_S3_BUCKET,
- buildAuditMap(s3BucketName), exception,
- getOmRequest().getUserInfo()));
-
- if (exception == null) {
- // Decrement s3 bucket and ozone bucket count. As S3 bucket is mapped to
- // ozonevolume/ozone bucket.
- LOG.debug("S3Bucket {} successfully deleted", s3BucketName);
- omMetrics.decNumS3Buckets();
- omMetrics.decNumBuckets();
-
- return omClientResponse;
- } else {
- LOG.error("S3Bucket Deletion failed for S3Bucket:{}", s3BucketName,
- exception);
- omMetrics.incNumS3BucketDeleteFails();
- return omClientResponse;
- }
- }
-
- /**
- * Extract volumeName from s3Mapping.
- * @param s3Mapping
- * @return volumeName
- * @throws IOException
- */
- private String getOzoneVolumeName(String s3Mapping) throws IOException {
- return s3Mapping.split("/")[0];
- }
-
- private Map<String, String> buildAuditMap(String s3BucketName) {
- Map<String, String> auditMap = new HashMap<>();
- auditMap.put(s3BucketName, OzoneConsts.S3_BUCKET);
- return auditMap;
- }
-
-}
\ No newline at end of file
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/bucket/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/bucket/package-info.java
deleted file mode 100644
index 7296585..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/bucket/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-/**
- * Package contains classes related to s3 bucket requests.
- */
-package org.apache.hadoop.ozone.om.request.s3.bucket;
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java
index 9a96810..f51cba8 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java
@@ -30,6 +30,7 @@
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
import org.apache.hadoop.ozone.om.request.key.OMKeyRequest;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.om.response.s3.multipart.S3InitiateMultipartUploadResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
@@ -109,10 +110,8 @@
Result result = null;
long objectID = OMFileRequest.getObjIDFromTxId(transactionLogIndex);
- OMResponse.Builder omResponse = OMResponse.newBuilder()
- .setCmdType(OzoneManagerProtocolProtos.Type.InitiateMultiPartUpload)
- .setStatus(OzoneManagerProtocolProtos.Status.OK)
- .setSuccess(true);
+ OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
+ getOmRequest());
OMClientResponse omClientResponse = null;
try {
// TODO to support S3 ACL later.
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java
index bec1b48..8c8e010 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java
@@ -22,6 +22,7 @@
import com.google.common.base.Optional;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -95,10 +96,8 @@
IOException exception = null;
OmMultipartKeyInfo multipartKeyInfo = null;
String multipartKey = null;
- OMResponse.Builder omResponse = OMResponse.newBuilder()
- .setCmdType(OzoneManagerProtocolProtos.Type.AbortMultiPartUpload)
- .setStatus(OzoneManagerProtocolProtos.Status.OK)
- .setSuccess(true);
+ OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
+ getOmRequest());
OMClientResponse omClientResponse = null;
Result result = null;
try {
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java
index 8120f12..d9004c0 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java
@@ -30,6 +30,7 @@
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.om.request.key.OMKeyRequest;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.om.response.s3.multipart
.S3MultipartUploadCommitPartResponse;
@@ -101,10 +102,8 @@
IOException exception = null;
String partName = null;
- OMResponse.Builder omResponse = OMResponse.newBuilder()
- .setCmdType(OzoneManagerProtocolProtos.Type.CommitMultiPartUpload)
- .setStatus(OzoneManagerProtocolProtos.Status.OK)
- .setSuccess(true);
+ OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
+ getOmRequest());
OMClientResponse omClientResponse = null;
OzoneManagerProtocolProtos.PartKeyInfo oldPartKeyInfo = null;
String openKey = null;
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
index 1514a20..17a8c61 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java
@@ -39,6 +39,7 @@
import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.om.request.key.OMKeyRequest;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.om.response.s3.multipart.S3MultipartUploadCompleteResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
@@ -118,10 +119,8 @@
keyName);
boolean acquiredLock = false;
- OMResponse.Builder omResponse = OMResponse.newBuilder()
- .setCmdType(OzoneManagerProtocolProtos.Type.CommitMultiPartUpload)
- .setStatus(OzoneManagerProtocolProtos.Status.OK)
- .setSuccess(true);
+ OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
+ getOmRequest());
OMClientResponse omClientResponse = null;
IOException exception = null;
Result result = null;
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/S3GetSecretRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/S3GetSecretRequest.java
index feb0e76..b240373 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/S3GetSecretRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/S3GetSecretRequest.java
@@ -24,6 +24,7 @@
import com.google.common.base.Optional;
import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,7 +40,6 @@
import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.om.response.s3.security.S3GetSecretResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetS3SecretRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetS3SecretResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
@@ -111,13 +111,9 @@
public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
long transactionLogIndex,
OzoneManagerDoubleBufferHelper ozoneManagerDoubleBufferHelper) {
-
-
OMClientResponse omClientResponse = null;
- OMResponse.Builder omResponse = OMResponse.newBuilder()
- .setCmdType(OzoneManagerProtocolProtos.Type.GetS3Secret)
- .setStatus(OzoneManagerProtocolProtos.Status.OK)
- .setSuccess(true);
+ OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
+ getOmRequest());
boolean acquiredLock = false;
IOException exception = null;
OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/security/OMCancelDelegationTokenRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/security/OMCancelDelegationTokenRequest.java
index 05ee99f..48451f5 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/security/OMCancelDelegationTokenRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/security/OMCancelDelegationTokenRequest.java
@@ -23,9 +23,9 @@
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.om.request.OMClientRequest;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.om.response.security.OMCancelDelegationTokenResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CancelDelegationTokenResponseProto;
@@ -71,11 +71,8 @@
OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
OMClientResponse omClientResponse = null;
- OMResponse.Builder omResponse =
- OMResponse.newBuilder()
- .setCmdType(OzoneManagerProtocolProtos.Type.CancelDelegationToken)
- .setStatus(OzoneManagerProtocolProtos.Status.OK)
- .setSuccess(true);
+ OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
+ getOmRequest());
OzoneTokenIdentifier ozoneTokenIdentifier = null;
try {
ozoneTokenIdentifier =
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/security/OMGetDelegationTokenRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/security/OMGetDelegationTokenRequest.java
index 39d9168..a3809be 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/security/OMGetDelegationTokenRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/security/OMGetDelegationTokenRequest.java
@@ -24,9 +24,9 @@
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.om.request.OMClientRequest;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.om.response.security.OMGetDelegationTokenResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetDelegationTokenResponseProto;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
@@ -117,15 +117,10 @@
UpdateGetDelegationTokenRequest updateGetDelegationTokenRequest =
getOmRequest().getUpdateGetDelegationTokenRequest();
- OMResponse.Builder omResponse =
- OMResponse.newBuilder()
- .setCmdType(OzoneManagerProtocolProtos.Type.GetDelegationToken)
- .setStatus(OzoneManagerProtocolProtos.Status.OK)
- .setSuccess(true);
-
+ OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
+ getOmRequest());
OMClientResponse omClientResponse = null;
-
// If security is not enabled and token request is received, leader
// returns token null. So, check here if updatedGetDelegationTokenResponse
// has response set or not. If it is not set, then token is null.
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/security/OMRenewDelegationTokenRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/security/OMRenewDelegationTokenRequest.java
index eb9c170..859b24b 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/security/OMRenewDelegationTokenRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/security/OMRenewDelegationTokenRequest.java
@@ -20,6 +20,7 @@
import java.io.IOException;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,7 +31,6 @@
import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.om.response.security.OMRenewDelegationTokenResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenewDelegationTokenResponseProto;
@@ -116,11 +116,8 @@
OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
OMClientResponse omClientResponse = null;
- OMResponse.Builder omResponse =
- OMResponse.newBuilder()
- .setCmdType(OzoneManagerProtocolProtos.Type.RenewDelegationToken)
- .setStatus(OzoneManagerProtocolProtos.Status.OK)
- .setSuccess(true);
+ OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
+ getOmRequest());
try {
OzoneTokenIdentifier ozoneTokenIdentifier = OzoneTokenIdentifier.
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/util/OmResponseUtil.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/util/OmResponseUtil.java
new file mode 100644
index 0000000..ba1dff6
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/util/OmResponseUtil.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.request.util;
+
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+
+/**
+ * Utility class to build OmResponse.
+ */
+public final class OmResponseUtil {
+
+ private OmResponseUtil() {
+ }
+
+ /**
+ * Get an initial OmResponse.Builder with proper request cmdType and traceID.
+ * @param request OMRequest.
+ * @return OMResponse builder.
+ */
+ public static OMResponse.Builder getOMResponseBuilder(OMRequest request) {
+ return OMResponse.newBuilder()
+ .setCmdType(request.getCmdType())
+ .setStatus(OzoneManagerProtocolProtos.Status.OK)
+ .setTraceID(request.getTraceID())
+ .setSuccess(true);
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeCreateRequest.java
index 47d8393..dcd66c7 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeCreateRequest.java
@@ -19,19 +19,22 @@
package org.apache.hadoop.ozone.om.request.volume;
import java.io.IOException;
-import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import com.google.common.base.Preconditions;
+
+import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
+import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.ozone.audit.OMAction;
import org.apache.hadoop.ozone.om.response.volume.OMVolumeCreateResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.om.OMMetrics;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OzoneManager;
@@ -51,7 +54,6 @@
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.UserVolumeInfo;
import org.apache.hadoop.util.Time;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD;
import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.VOLUME_LOCK;
import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.USER_LOCK;
@@ -71,17 +73,17 @@
VolumeInfo volumeInfo =
getOmRequest().getCreateVolumeRequest().getVolumeInfo();
+ // Verify resource name
+ OmUtils.validateVolumeName(volumeInfo.getVolume());
// Set creation time
VolumeInfo updatedVolumeInfo =
volumeInfo.toBuilder().setCreationTime(Time.now()).build();
-
return getOmRequest().toBuilder().setCreateVolumeRequest(
CreateVolumeRequest.newBuilder().setVolumeInfo(updatedVolumeInfo))
.setUserInfo(getUserInfo())
.build();
-
}
@Override
@@ -100,9 +102,8 @@
String volume = volumeInfo.getVolume();
String owner = volumeInfo.getOwnerName();
- OMResponse.Builder omResponse = OMResponse.newBuilder().setCmdType(
- OzoneManagerProtocolProtos.Type.CreateVolume).setStatus(
- OzoneManagerProtocolProtos.Status.OK).setSuccess(true);
+ OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
+ getOmRequest());
OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
@@ -113,7 +114,6 @@
OMClientResponse omClientResponse = null;
OmVolumeArgs omVolumeArgs = null;
Map<String, String> auditMap = new HashMap<>();
- Collection<String> ozAdmins = ozoneManager.getOzoneAdmins();
try {
omVolumeArgs = OmVolumeArgs.getFromProtobuf(volumeInfo);
// when you create a volume, we set both Object ID and update ID.
@@ -127,14 +127,11 @@
auditMap = omVolumeArgs.toAuditMap();
- // check Acl
+ // check acl
if (ozoneManager.getAclsEnabled()) {
- if (!ozAdmins.contains(OZONE_ADMINISTRATORS_WILDCARD) &&
- !ozAdmins.contains(getUserInfo().getUserName())) {
- throw new OMException("Only admin users are authorized to create " +
- "Ozone volumes. User: " + getUserInfo().getUserName(),
- OMException.ResultCodes.PERMISSION_DENIED);
- }
+ checkAcls(ozoneManager, OzoneObj.ResourceType.VOLUME,
+ OzoneObj.StoreType.OZONE, IAccessAuthorizer.ACLType.CREATE, volume,
+ null, null);
}
// acquire lock.
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeDeleteRequest.java
index 88b11b9..4d2f055 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeDeleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeDeleteRequest.java
@@ -23,6 +23,7 @@
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -76,9 +77,8 @@
OMMetrics omMetrics = ozoneManager.getMetrics();
omMetrics.incNumVolumeDeletes();
- OMResponse.Builder omResponse = OMResponse.newBuilder().setCmdType(
- OzoneManagerProtocolProtos.Type.DeleteVolume).setStatus(
- OzoneManagerProtocolProtos.Status.OK).setSuccess(true);
+ OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
+ getOmRequest());
OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
boolean acquiredUserLock = false;
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeRequest.java
index bbd0480..4c481a1 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeRequest.java
@@ -29,16 +29,23 @@
.UserVolumeInfo;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
/**
* Defines common methods required for volume requests.
*/
public abstract class OMVolumeRequest extends OMClientRequest {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(OMVolumeRequest.class);
+
public OMVolumeRequest(OMRequest omRequest) {
super(omRequest);
}
@@ -99,22 +106,18 @@
OMException.ResultCodes.USER_TOO_MANY_VOLUMES);
}
- List<String> prevVolList = new ArrayList<>();
+ Set<String> volumeSet = new HashSet<>();
long objectID = txID;
if (volumeList != null) {
- prevVolList.addAll(volumeList.getVolumeNamesList());
+ volumeSet.addAll(volumeList.getVolumeNamesList());
objectID = volumeList.getObjectID();
}
-
- // Add the new volume to the list
- prevVolList.add(volume);
- UserVolumeInfo newVolList = UserVolumeInfo.newBuilder()
+ volumeSet.add(volume);
+ return UserVolumeInfo.newBuilder()
.setObjectID(objectID)
.setUpdateID(txID)
- .addAllVolumeNames(prevVolList).build();
-
- return newVolList;
+ .addAllVolumeNames(volumeSet).build();
}
/**
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetOwnerRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetOwnerRequest.java
index 6b603e5..898f345 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetOwnerRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetOwnerRequest.java
@@ -22,6 +22,7 @@
import java.util.Map;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Optional;
@@ -71,13 +72,10 @@
SetVolumePropertyRequest setVolumePropertyRequest =
getOmRequest().getSetVolumePropertyRequest();
-
Preconditions.checkNotNull(setVolumePropertyRequest);
- OMResponse.Builder omResponse = OMResponse.newBuilder().setCmdType(
- OzoneManagerProtocolProtos.Type.SetVolumeProperty).setStatus(
- OzoneManagerProtocolProtos.Status.OK).setSuccess(true);
-
+ OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
+ getOmRequest());
// In production this will never happen, this request will be called only
// when we have ownerName in setVolumePropertyRequest.
if (!setVolumePropertyRequest.hasOwnerName()) {
@@ -112,18 +110,14 @@
}
long maxUserVolumeCount = ozoneManager.getMaxUserVolumeCount();
-
String dbVolumeKey = omMetadataManager.getVolumeKey(volume);
-
OzoneManagerProtocolProtos.UserVolumeInfo oldOwnerVolumeList = null;
OzoneManagerProtocolProtos.UserVolumeInfo newOwnerVolumeList = null;
OmVolumeArgs omVolumeArgs = null;
acquiredVolumeLock = omMetadataManager.getLock().acquireWriteLock(
VOLUME_LOCK, volume);
-
omVolumeArgs = omMetadataManager.getVolumeTable().get(dbVolumeKey);
-
if (omVolumeArgs == null) {
LOG.debug("Changing volume ownership failed for user:{} volume:{}",
newOwner, volume);
@@ -143,15 +137,26 @@
oldOwner = omVolumeArgs.getOwnerName();
+ // Return OK immediately if newOwner is the same as oldOwner.
+ if (oldOwner.equals(newOwner)) {
+ LOG.warn("Volume '{}' owner is already user '{}'.", volume, oldOwner);
+ omResponse.setStatus(OzoneManagerProtocolProtos.Status.OK)
+ .setMessage(
+ "Volume '" + volume + "' owner is already '" + newOwner + "'.")
+ .setSuccess(false);
+ omResponse.setSetVolumePropertyResponse(
+ SetVolumePropertyResponse.newBuilder().setResponse(false).build());
+ omClientResponse = new OMVolumeSetOwnerResponse(omResponse.build());
+ // Note: addResponseToDoubleBuffer would be executed in finally block.
+ return omClientResponse;
+ }
+
acquiredUserLocks =
omMetadataManager.getLock().acquireMultiUserLock(newOwner, oldOwner);
-
oldOwnerVolumeList =
omMetadataManager.getUserTable().get(oldOwner);
-
oldOwnerVolumeList = delVolumeFromOwnerList(
oldOwnerVolumeList, volume, oldOwner, transactionLogIndex);
-
newOwnerVolumeList = omMetadataManager.getUserTable().get(newOwner);
newOwnerVolumeList = addVolumeToOwnerList(
newOwnerVolumeList, volume, newOwner,
@@ -165,8 +170,8 @@
// Update cache.
omMetadataManager.getUserTable().addCacheEntry(
new CacheKey<>(omMetadataManager.getUserKey(newOwner)),
- new CacheValue<>(Optional.of(newOwnerVolumeList),
- transactionLogIndex));
+ new CacheValue<>(Optional.of(newOwnerVolumeList),
+ transactionLogIndex));
omMetadataManager.getUserTable().addCacheEntry(
new CacheKey<>(omMetadataManager.getUserKey(oldOwner)),
new CacheValue<>(Optional.of(oldOwnerVolumeList),
@@ -176,7 +181,7 @@
new CacheValue<>(Optional.of(omVolumeArgs), transactionLogIndex));
omResponse.setSetVolumePropertyResponse(
- SetVolumePropertyResponse.newBuilder().build());
+ SetVolumePropertyResponse.newBuilder().setResponse(true).build());
omClientResponse = new OMVolumeSetOwnerResponse(omResponse.build(),
oldOwner, oldOwnerVolumeList, newOwnerVolumeList, omVolumeArgs);
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetQuotaRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetQuotaRequest.java
index ff5ecd3..91b02a2 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetQuotaRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetQuotaRequest.java
@@ -24,6 +24,7 @@
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -74,9 +75,8 @@
Preconditions.checkNotNull(setVolumePropertyRequest);
- OMResponse.Builder omResponse = OMResponse.newBuilder().setCmdType(
- OzoneManagerProtocolProtos.Type.SetVolumeProperty).setStatus(
- OzoneManagerProtocolProtos.Status.OK).setSuccess(true);
+ OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
+ getOmRequest());
// In production this will never happen, this request will be called only
// when we have quota in bytes is set in setVolumePropertyRequest.
@@ -93,7 +93,7 @@
AuditLogger auditLogger = ozoneManager.getAuditLogger();
OzoneManagerProtocolProtos.UserInfo userInfo = getOmRequest().getUserInfo();
Map<String, String> auditMap = buildVolumeAuditMap(volume);
- auditMap.put(OzoneConsts.QUOTA,
+ auditMap.put(OzoneConsts.QUOTA_IN_BYTES,
String.valueOf(setVolumePropertyRequest.getQuotaInBytes()));
OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeAddAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeAddAclRequest.java
index 0fd8302..e0f9b3d 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeAddAclRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeAddAclRequest.java
@@ -22,6 +22,7 @@
import org.apache.hadoop.hdds.scm.storage.CheckedBiFunction;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.om.response.volume.OMVolumeAclOpResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
@@ -77,9 +78,7 @@
@Override
OMResponse.Builder onInit() {
- return OMResponse.newBuilder().setCmdType(
- OzoneManagerProtocolProtos.Type.AddAcl)
- .setStatus(OzoneManagerProtocolProtos.Status.OK).setSuccess(true);
+ return OmResponseUtil.getOMResponseBuilder(getOmRequest());
}
@Override
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeRemoveAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeRemoveAclRequest.java
index 5cadd1b..6e90731 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeRemoveAclRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeRemoveAclRequest.java
@@ -22,6 +22,7 @@
import org.apache.hadoop.hdds.scm.storage.CheckedBiFunction;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.om.response.volume.OMVolumeAclOpResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
@@ -76,9 +77,7 @@
@Override
OMResponse.Builder onInit() {
- return OMResponse.newBuilder().setCmdType(
- OzoneManagerProtocolProtos.Type.RemoveAcl)
- .setStatus(OzoneManagerProtocolProtos.Status.OK).setSuccess(true);
+ return OmResponseUtil.getOMResponseBuilder(getOmRequest());
}
@Override
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeSetAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeSetAclRequest.java
index 49b98bd..8d5bc61 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeSetAclRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeSetAclRequest.java
@@ -21,6 +21,7 @@
import org.apache.hadoop.hdds.scm.storage.CheckedBiFunction;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.om.response.volume.OMVolumeAclOpResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
@@ -73,9 +74,7 @@
@Override
OMResponse.Builder onInit() {
- return OMResponse.newBuilder().setCmdType(
- OzoneManagerProtocolProtos.Type.RemoveAcl)
- .setStatus(OzoneManagerProtocolProtos.Status.OK).setSuccess(true);
+ return OmResponseUtil.getOMResponseBuilder(getOmRequest());
}
@Override
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMTrashRecoverResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMTrashRecoverResponse.java
new file mode 100644
index 0000000..fb330a3
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMTrashRecoverResponse.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.response.key;
+
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .OMResponse;
+import org.apache.hadoop.hdds.utils.db.BatchOperation;
+
+import java.io.IOException;
+import javax.annotation.Nullable;
+import javax.annotation.Nonnull;
+
+/**
+ * Response for RecoverTrash request.
+ */
+public class OMTrashRecoverResponse extends OMClientResponse {
+ private OmKeyInfo omKeyInfo;
+
+ public OMTrashRecoverResponse(@Nullable OmKeyInfo omKeyInfo,
+ @Nonnull OMResponse omResponse) {
+ super(omResponse);
+ this.omKeyInfo = omKeyInfo;
+ }
+
+ @Override
+ public void addToDBBatch(OMMetadataManager omMetadataManager,
+ BatchOperation batchOperation) throws IOException {
+
+ /* TODO: HDDS-2425. HDDS-2426. */
+ String trashKey = omMetadataManager
+ .getOzoneKey(omKeyInfo.getVolumeName(),
+ omKeyInfo.getBucketName(), omKeyInfo.getKeyName());
+ RepeatedOmKeyInfo repeatedOmKeyInfo = omMetadataManager
+ .getDeletedTable().get(trashKey);
+ omKeyInfo = OmUtils.prepareKeyForRecover(omKeyInfo, repeatedOmKeyInfo);
+ omMetadataManager.getDeletedTable()
+ .deleteWithBatch(batchOperation, omKeyInfo.getKeyName());
+ /* TODO: trashKey should be updated to destinationBucket. */
+ omMetadataManager.getKeyTable()
+ .putWithBatch(batchOperation, trashKey, omKeyInfo);
+ }
+
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/bucket/S3BucketCreateResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/bucket/S3BucketCreateResponse.java
deleted file mode 100644
index 7daf9e8..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/bucket/S3BucketCreateResponse.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.om.response.s3.bucket;
-
-import javax.annotation.Nonnull;
-import java.io.IOException;
-
-import com.google.common.base.Preconditions;
-import com.google.common.annotations.VisibleForTesting;
-
-import javax.annotation.Nullable;
-import org.apache.hadoop.ozone.om.OMMetadataManager;
-import org.apache.hadoop.ozone.om.response.OMClientResponse;
-import org.apache.hadoop.ozone.om.response.bucket.OMBucketCreateResponse;
-import org.apache.hadoop.ozone.om.response.volume.OMVolumeCreateResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
- .OMResponse;
-import org.apache.hadoop.hdds.utils.db.BatchOperation;
-
-/**
- * Response for S3Bucket create request.
- */
-public class S3BucketCreateResponse extends OMClientResponse {
-
- private OMVolumeCreateResponse omVolumeCreateResponse;
- private OMBucketCreateResponse omBucketCreateResponse;
- private String s3Bucket;
- private String s3Mapping;
-
- public S3BucketCreateResponse(@Nonnull OMResponse omResponse,
- @Nullable OMVolumeCreateResponse omVolumeCreateResponse,
- @Nonnull OMBucketCreateResponse omBucketCreateResponse,
- @Nonnull String s3BucketName,
- @Nonnull String s3Mapping) {
- super(omResponse);
- this.omVolumeCreateResponse = omVolumeCreateResponse;
- this.omBucketCreateResponse = omBucketCreateResponse;
- this.s3Bucket = s3BucketName;
- this.s3Mapping = s3Mapping;
- }
-
- /**
- * For when the request is not successful or it is a replay transaction.
- * For a successful request, the other constructor should be used.
- */
- public S3BucketCreateResponse(@Nonnull OMResponse omResponse) {
- super(omResponse);
- checkStatusNotOK();
- }
-
- @Override
- protected void addToDBBatch(OMMetadataManager omMetadataManager,
- BatchOperation batchOperation) throws IOException {
-
- if (omVolumeCreateResponse != null) {
- omVolumeCreateResponse.checkAndUpdateDB(omMetadataManager,
- batchOperation);
- }
-
- Preconditions.checkState(omBucketCreateResponse != null);
- omBucketCreateResponse.checkAndUpdateDB(omMetadataManager, batchOperation);
-
- omMetadataManager.getS3Table().putWithBatch(batchOperation, s3Bucket,
- s3Mapping);
- }
-
- @VisibleForTesting
- public String getS3Mapping() {
- return s3Mapping;
- }
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/bucket/S3BucketDeleteResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/bucket/S3BucketDeleteResponse.java
deleted file mode 100644
index c4c4a0c..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/bucket/S3BucketDeleteResponse.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.om.response.s3.bucket;
-
-import org.apache.hadoop.ozone.om.OMMetadataManager;
-import org.apache.hadoop.ozone.om.response.OMClientResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
-import org.apache.hadoop.hdds.utils.db.BatchOperation;
-
-import javax.annotation.Nonnull;
-import java.io.IOException;
-
-/**
- * Response for S3Bucket Delete request.
- */
-public class S3BucketDeleteResponse extends OMClientResponse {
-
- private String s3BucketName;
- private String volumeName;
-
- public S3BucketDeleteResponse(@Nonnull OMResponse omResponse,
- @Nonnull String s3BucketName, @Nonnull String volumeName) {
- super(omResponse);
- this.s3BucketName = s3BucketName;
- this.volumeName = volumeName;
- }
-
- /**
- * For when the request is not successful or it is a replay transaction.
- * For a successful request, the other constructor should be used.
- */
- public S3BucketDeleteResponse(@Nonnull OMResponse omResponse) {
- super(omResponse);
- checkStatusNotOK();
- }
-
- @Override
- public void addToDBBatch(OMMetadataManager omMetadataManager,
- BatchOperation batchOperation) throws IOException {
-
- omMetadataManager.getBucketTable().deleteWithBatch(batchOperation,
- omMetadataManager.getBucketKey(volumeName, s3BucketName));
- omMetadataManager.getS3Table().deleteWithBatch(batchOperation,
- s3BucketName);
- }
-}
\ No newline at end of file
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/volume/OMVolumeSetOwnerResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/volume/OMVolumeSetOwnerResponse.java
index 8cd1f05..469f1ae 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/volume/OMVolumeSetOwnerResponse.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/volume/OMVolumeSetOwnerResponse.java
@@ -24,6 +24,7 @@
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.UserVolumeInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
@@ -55,11 +56,27 @@
/**
* For when the request is not successful or it is a replay transaction.
- * For a successful request, the other constructor should be used.
+ * Or when newOwner is the same as oldOwner.
+ * For other successful requests, the other constructor should be used.
*/
public OMVolumeSetOwnerResponse(@Nonnull OMResponse omResponse) {
super(omResponse);
- checkStatusNotOK();
+ // When newOwner is the same as oldOwner, status is OK but success is false.
+ // We want to bypass the check in this case.
+ if (omResponse.getSuccess()) {
+ checkStatusNotOK();
+ }
+ }
+
+ @Override
+ public void checkAndUpdateDB(OMMetadataManager omMetadataManager,
+ BatchOperation batchOperation) throws IOException {
+ // When newOwner is the same as oldOwner, status is OK but success is false.
+ // We don't want to add it to DB batch in this case.
+ if (getOMResponse().getStatus() == OzoneManagerProtocolProtos.Status.OK &&
+ getOMResponse().getSuccess()) {
+ addToDBBatch(omMetadataManager, batchOperation);
+ }
}
@Override
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OzoneManagerSnapshotProvider.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OzoneManagerSnapshotProvider.java
index 642bbcd..8fcf4e9 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OzoneManagerSnapshotProvider.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OzoneManagerSnapshotProvider.java
@@ -21,6 +21,8 @@
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
@@ -28,11 +30,13 @@
import java.util.Map;
import java.util.concurrent.TimeUnit;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.server.http.HttpConfig;
+import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
import org.apache.hadoop.hdds.utils.db.RocksDBCheckpoint;
+import org.apache.hadoop.hdfs.web.URLConnectionFactory;
import org.apache.hadoop.ozone.om.ha.OMNodeDetails;
import static java.net.HttpURLConnection.HTTP_CREATED;
@@ -40,21 +44,13 @@
import org.apache.commons.io.FileUtils;
import static org.apache.hadoop.ozone.OzoneConsts.OM_RATIS_SNAPSHOT_INDEX;
import static org.apache.hadoop.ozone.OzoneConsts.OM_RATIS_SNAPSHOT_TERM;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_AUTH_TYPE;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_PROVIDER_CONNECTION_TIMEOUT_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_PROVIDER_CONNECTION_TIMEOUT_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_PROVIDER_REQUEST_TIMEOUT_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_PROVIDER_REQUEST_TIMEOUT_KEY;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_PROVIDER_SOCKET_TIMEOUT_DEFAULT;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_PROVIDER_SOCKET_TIMEOUT_KEY;
-import org.apache.http.Header;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.http.util.EntityUtils;
+
+import org.apache.hadoop.security.SecurityUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,12 +66,12 @@
private final File omSnapshotDir;
private Map<String, OMNodeDetails> peerNodesMap;
private final HttpConfig.Policy httpPolicy;
- private final RequestConfig httpRequestConfig;
- private CloseableHttpClient httpClient;
+ private final boolean spnegoEnabled;
+ private final URLConnectionFactory connectionFactory;
private static final String OM_SNAPSHOT_DB = "om.snapshot.db";
- public OzoneManagerSnapshotProvider(Configuration conf,
+ public OzoneManagerSnapshotProvider(ConfigurationSource conf,
File omRatisSnapshotDir, List<OMNodeDetails> peerNodes) {
LOG.info("Initializing OM Snapshot Provider");
@@ -87,16 +83,8 @@
}
this.httpPolicy = HttpConfig.getHttpPolicy(conf);
- this.httpRequestConfig = getHttpRequestConfig(conf);
- }
-
- private RequestConfig getHttpRequestConfig(Configuration conf) {
- TimeUnit socketTimeoutUnit =
- OZONE_OM_SNAPSHOT_PROVIDER_SOCKET_TIMEOUT_DEFAULT.getUnit();
- int socketTimeoutMS = (int) conf.getTimeDuration(
- OZONE_OM_SNAPSHOT_PROVIDER_SOCKET_TIMEOUT_KEY,
- OZONE_OM_SNAPSHOT_PROVIDER_SOCKET_TIMEOUT_DEFAULT.getDuration(),
- socketTimeoutUnit);
+ this.spnegoEnabled = conf.get(OZONE_OM_HTTP_AUTH_TYPE, "simple")
+ .equals("kerberos");
TimeUnit connectionTimeoutUnit =
OZONE_OM_SNAPSHOT_PROVIDER_CONNECTION_TIMEOUT_DEFAULT.getUnit();
@@ -112,36 +100,9 @@
OZONE_OM_SNAPSHOT_PROVIDER_REQUEST_TIMEOUT_DEFAULT.getDuration(),
requestTimeoutUnit);
- RequestConfig requestConfig = RequestConfig.custom()
- .setSocketTimeout(socketTimeoutMS)
- .setConnectTimeout(connectionTimeoutMS)
- .setConnectionRequestTimeout(requestTimeoutMS)
- .build();
-
- return requestConfig;
- }
-
- /**
- * Create and return http client object.
- */
- private HttpClient getHttpClient() {
- if (httpClient == null) {
- httpClient = HttpClientBuilder
- .create()
- .setDefaultRequestConfig(httpRequestConfig)
- .build();
- }
- return httpClient;
- }
-
- /**
- * Close http client object.
- */
- private void closeHttpClient() throws IOException {
- if (httpClient != null) {
- httpClient.close();
- httpClient = null;
- }
+ connectionFactory = URLConnectionFactory
+ .newDefaultURLConnectionFactory(connectionTimeoutMS, requestTimeoutMS,
+ LegacyHadoopConfigurationSource.asHadoopConfiguration(conf));
}
/**
@@ -159,61 +120,55 @@
LOG.info("Downloading latest checkpoint from Leader OM {}. Checkpoint " +
"URL: {}", leaderOMNodeID, omCheckpointUrl);
-
- try {
- HttpGet httpGet = new HttpGet(omCheckpointUrl);
- HttpResponse response = getHttpClient().execute(httpGet);
- int errorCode = response.getStatusLine().getStatusCode();
- HttpEntity entity = response.getEntity();
-
- if ((errorCode == HTTP_OK) || (errorCode == HTTP_CREATED)) {
-
- Header header = response.getFirstHeader(OM_RATIS_SNAPSHOT_INDEX);
- if (header == null) {
- throw new IOException("The HTTP response header " +
- OM_RATIS_SNAPSHOT_INDEX + " is missing.");
- }
-
- long snapshotIndex = Long.parseLong(header.getValue());
-
- header = response.getFirstHeader(OM_RATIS_SNAPSHOT_TERM);
- if (header == null) {
- throw new IOException("The HTTP response header " +
- OM_RATIS_SNAPSHOT_TERM + " is missing.");
- }
-
- long snapshotTerm = Long.parseLong(header.getValue());
-
- try (InputStream inputStream = entity.getContent()) {
- FileUtils.copyInputStreamToFile(inputStream, targetFile);
- }
-
- // Untar the checkpoint file.
- Path untarredDbDir = Paths.get(omSnapshotDir.getAbsolutePath(),
- snapshotFileName);
- FileUtil.unTar(targetFile, untarredDbDir.toFile());
- FileUtils.deleteQuietly(targetFile);
-
- LOG.info("Sucessfully downloaded latest checkpoint with snapshot " +
- "index {} from leader OM: {}", snapshotIndex, leaderOMNodeID);
-
- RocksDBCheckpoint omCheckpoint = new RocksDBCheckpoint(untarredDbDir);
- omCheckpoint.setRatisSnapshotIndex(snapshotIndex);
- omCheckpoint.setRatisSnapshotTerm(snapshotTerm);
- return omCheckpoint;
- }
-
- if (entity != null) {
+ final long[] snapshotIndex = new long[1];
+ final long[] snapshotTerm = new long[1];
+ SecurityUtil.doAsCurrentUser(() -> {
+ HttpURLConnection httpURLConnection = (HttpURLConnection)
+ connectionFactory.openConnection(new URL(omCheckpointUrl),
+ spnegoEnabled);
+ httpURLConnection.connect();
+ int errorCode = httpURLConnection.getResponseCode();
+ if ((errorCode != HTTP_OK) && (errorCode != HTTP_CREATED)) {
throw new IOException("Unexpected exception when trying to reach " +
"OM to download latest checkpoint. Checkpoint URL: " +
- omCheckpointUrl + ". Entity: " + EntityUtils.toString(entity));
- } else {
- throw new IOException("Unexpected null in http payload, while " +
- "processing request to OM to download latest checkpoint. " +
- "Checkpoint Url: " + omCheckpointUrl);
+ omCheckpointUrl + ". ErrorCode: " + errorCode);
}
- } finally {
- closeHttpClient();
+ snapshotIndex[0] = httpURLConnection.getHeaderFieldLong(
+ OM_RATIS_SNAPSHOT_INDEX, -1);
+ if (snapshotIndex[0] == -1) {
+ throw new IOException("The HTTP response header " +
+ OM_RATIS_SNAPSHOT_INDEX + " is missing.");
+ }
+ snapshotTerm[0] = httpURLConnection.getHeaderFieldLong(
+ OM_RATIS_SNAPSHOT_TERM, -1);
+ if (snapshotTerm[0] == -1) {
+ throw new IOException("The HTTP response header " +
+ OM_RATIS_SNAPSHOT_TERM + " is missing.");
+ }
+
+ try (InputStream inputStream = httpURLConnection.getInputStream()) {
+ FileUtils.copyInputStreamToFile(inputStream, targetFile);
+ }
+ return null;
+ });
+ // Untar the checkpoint file.
+ Path untarredDbDir = Paths.get(omSnapshotDir.getAbsolutePath(),
+ snapshotFileName);
+ FileUtil.unTar(targetFile, untarredDbDir.toFile());
+ FileUtils.deleteQuietly(targetFile);
+
+ LOG.info("Sucessfully downloaded latest checkpoint with snapshot " +
+ "index {} from leader OM: {}", snapshotIndex[0], leaderOMNodeID);
+
+ RocksDBCheckpoint omCheckpoint = new RocksDBCheckpoint(untarredDbDir);
+ omCheckpoint.setRatisSnapshotIndex(snapshotIndex[0]);
+ omCheckpoint.setRatisSnapshotTerm(snapshotTerm[0]);
+ return omCheckpoint;
+ }
+
+ public void stop() {
+ if (connectionFactory != null) {
+ connectionFactory.destroy();
}
}
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
index 7ee2e6e..6a6cdc4 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
@@ -16,7 +16,13 @@
*/
package org.apache.hadoop.ozone.protocolPB;
+import java.io.IOException;
+import java.util.Optional;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicLong;
+
import org.apache.hadoop.hdds.server.OzoneProtocolMessageDispatcher;
+import org.apache.hadoop.hdds.tracing.TracingUtil;
import org.apache.hadoop.hdds.utils.ProtocolMessageMetrics;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.om.OzoneManager;
@@ -27,10 +33,10 @@
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils;
import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+import com.google.protobuf.ProtocolMessageEnum;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
import org.apache.ratis.protocol.RaftPeerId;
@@ -38,11 +44,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.Optional;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.atomic.AtomicLong;
-
/**
* This class is the server-side translator that forwards requests received on
* {@link OzoneManagerProtocolPB}
@@ -69,16 +70,20 @@
public OzoneManagerProtocolServerSideTranslatorPB(
OzoneManager impl,
OzoneManagerRatisServer ratisServer,
- ProtocolMessageMetrics metrics,
+ ProtocolMessageMetrics<ProtocolMessageEnum> metrics,
boolean enableRatis) {
this.ozoneManager = impl;
this.isRatisEnabled = enableRatis;
- this.ozoneManagerDoubleBuffer =
- new OzoneManagerDoubleBuffer(ozoneManager.getMetadataManager(), (i) -> {
- // Do nothing.
- // For OM NON-HA code, there is no need to save transaction index.
- // As we wait until the double buffer flushes DB to disk.
- }, isRatisEnabled);
+ this.ozoneManagerDoubleBuffer = new OzoneManagerDoubleBuffer.Builder()
+ .setOmMetadataManager(ozoneManager.getMetadataManager())
+ // Do nothing.
+ // For OM NON-HA code, there is no need to save transaction index.
+ // As we wait until the double buffer flushes DB to disk.
+ .setOzoneManagerRatisSnapShot((i) -> {})
+ .enableRatis(isRatisEnabled)
+ .enableTracing(TracingUtil.isTracingEnabled(
+ ozoneManager.getConfiguration()))
+ .build();
handler = new OzoneManagerRequestHandler(impl, ozoneManagerDoubleBuffer);
this.omRatisServer = ratisServer;
dispatcher = new OzoneProtocolMessageDispatcher<>("OzoneProtocol",
@@ -140,14 +145,13 @@
*/
private OMResponse createErrorResponse(
OMRequest omRequest, IOException exception) {
- OzoneManagerProtocolProtos.Type cmdType = omRequest.getCmdType();
// Added all write command types here, because in future if any of the
// preExecute is changed to return IOException, we can return the error
// OMResponse to the client.
OMResponse.Builder omResponse = OMResponse.newBuilder()
- .setStatus(
- OzoneManagerRatisUtils.exceptionToResponseStatus(exception))
- .setCmdType(cmdType)
+ .setStatus(OzoneManagerRatisUtils.exceptionToResponseStatus(exception))
+ .setCmdType(omRequest.getCmdType())
+ .setTraceID(omRequest.getTraceID())
.setSuccess(false);
if (exception.getMessage() != null) {
omResponse.setMessage(exception.getMessage());
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
index 03b8715..e7b0265 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
@@ -23,9 +23,11 @@
import java.util.stream.Collectors;
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
+import org.apache.hadoop.hdds.utils.db.SequenceNumberNotFoundException;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.DBUpdates;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
@@ -41,13 +43,14 @@
import org.apache.hadoop.ozone.om.ratis.OzoneManagerDoubleBuffer;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils;
import org.apache.hadoop.ozone.om.request.OMClientRequest;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetFileStatusRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetFileStatusResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.AllocateBlockRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.AllocateBlockResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CheckVolumeAccessRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CheckVolumeAccessResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetFileStatusRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetFileStatusResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.InfoBucketRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.InfoBucketResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.InfoVolumeRequest;
@@ -67,10 +70,6 @@
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.MultipartUploadListPartsResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3BucketInfoRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3BucketInfoResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3ListBucketsRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3ListBucketsResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status;
@@ -78,14 +77,22 @@
import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
import com.google.common.collect.Lists;
-
-import org.apache.hadoop.hdds.utils.db.DBUpdatesWrapper;
-import org.apache.hadoop.hdds.utils.db.SequenceNumberNotFoundException;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DBUpdatesRequest;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DBUpdatesResponse;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetAclRequest;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetAclResponse;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListMultipartUploadsRequest;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListMultipartUploadsResponse;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListStatusRequest;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListStatusResponse;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LookupFileRequest;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LookupFileResponse;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.MultipartUploadInfo;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.*;
-
/**
* Command Handler for OM requests. OM State Machine calls this handler for
* deserializing the client request and sending it to OM.
@@ -110,9 +117,8 @@
LOG.debug("Received OMRequest: {}, ", request);
}
Type cmdType = request.getCmdType();
- OMResponse.Builder responseBuilder = OMResponse.newBuilder()
- .setCmdType(cmdType)
- .setStatus(Status.OK);
+ OMResponse.Builder responseBuilder = OmResponseUtil.getOMResponseBuilder(
+ request);
try {
switch (cmdType) {
case CheckVolumeAccess:
@@ -155,16 +161,6 @@
request.getListTrashRequest());
responseBuilder.setListTrashResponse(listTrashResponse);
break;
- case InfoS3Bucket:
- S3BucketInfoResponse s3BucketInfoResponse = getS3Bucketinfo(
- request.getInfoS3BucketRequest());
- responseBuilder.setInfoS3BucketResponse(s3BucketInfoResponse);
- break;
- case ListS3Buckets:
- S3ListBucketsResponse s3ListBucketsResponse = listS3Buckets(
- request.getListS3BucketsRequest());
- responseBuilder.setListS3BucketsResponse(s3ListBucketsResponse);
- break;
case ListMultiPartUploadParts:
MultipartUploadListPartsResponse listPartsResponse =
listParts(request.getListMultipartUploadPartsRequest());
@@ -243,7 +239,7 @@
DBUpdatesResponse.Builder builder = DBUpdatesResponse
.newBuilder();
- DBUpdatesWrapper dbUpdatesWrapper =
+ DBUpdates dbUpdatesWrapper =
impl.getDBUpdates(dbUpdatesRequest);
for (int i = 0; i < dbUpdatesWrapper.getData().size(); i++) {
builder.addData(OMPBHelper.getByteString(
@@ -461,31 +457,6 @@
return resp.build();
}
- private S3BucketInfoResponse getS3Bucketinfo(S3BucketInfoRequest request)
- throws IOException {
- S3BucketInfoResponse.Builder resp = S3BucketInfoResponse.newBuilder();
-
- resp.setOzoneMapping(
- impl.getOzoneBucketMapping(request.getS3BucketName()));
- return resp.build();
- }
-
- private S3ListBucketsResponse listS3Buckets(S3ListBucketsRequest request)
- throws IOException {
- S3ListBucketsResponse.Builder resp = S3ListBucketsResponse.newBuilder();
-
- List<OmBucketInfo> buckets = impl.listS3Buckets(
- request.getUserName(),
- request.getStartKey(),
- request.getPrefix(),
- request.getCount());
- for (OmBucketInfo bucket : buckets) {
- resp.addBucketInfo(bucket.getProtobuf());
- }
-
- return resp.build();
- }
-
private MultipartUploadListPartsResponse listParts(
MultipartUploadListPartsRequest multipartUploadListPartsRequest)
throws IOException {
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/AWSV4AuthValidator.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/AWSV4AuthValidator.java
similarity index 100%
rename from hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/AWSV4AuthValidator.java
rename to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/AWSV4AuthValidator.java
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSecretManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSecretManager.java
similarity index 100%
rename from hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSecretManager.java
rename to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSecretManager.java
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java
similarity index 88%
rename from hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java
rename to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java
index 6e00da6..2f3e2ff 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java
@@ -66,6 +66,8 @@
private Thread tokenRemoverThread;
private final long tokenRemoverScanInterval;
private String omCertificateSerialId;
+ private String omServiceId;
+
/**
* If the delegation token update thread holds this lock, it will not get
* interrupted.
@@ -75,34 +77,82 @@
private boolean isRatisEnabled;
/**
- * Create a secret manager.
+ * Create a secret manager with a builder object.
*
- * @param conf configuration.
- * @param tokenMaxLifetime the maximum lifetime of the delegation tokens in
- * milliseconds
- * @param tokenRenewInterval how often the tokens must be renewed in
- * milliseconds
- * @param dtRemoverScanInterval how often the tokens are scanned for expired
- * tokens in milliseconds
- * @param certClient certificate client to SCM CA
- */
- public OzoneDelegationTokenSecretManager(OzoneConfiguration conf,
- long tokenMaxLifetime, long tokenRenewInterval,
- long dtRemoverScanInterval, Text service,
- S3SecretManager s3SecretManager, CertificateClient certClient)
- throws IOException {
- super(new SecurityConfig(conf), tokenMaxLifetime, tokenRenewInterval,
- service, LOG);
- setCertClient(certClient);
+ **/
+ public OzoneDelegationTokenSecretManager(Builder b) throws IOException {
+ super(new SecurityConfig(b.ozoneConf), b.tokenMaxLifetime,
+ b.tokenRenewInterval, b.service, LOG);
+ setCertClient(b.certClient);
+ this.omServiceId = b.omServiceId;
currentTokens = new ConcurrentHashMap();
- this.tokenRemoverScanInterval = dtRemoverScanInterval;
- this.s3SecretManager = (S3SecretManagerImpl) s3SecretManager;
- this.store = new OzoneSecretStore(conf,
+ this.tokenRemoverScanInterval = b.tokenRemoverScanInterval;
+ this.s3SecretManager = (S3SecretManagerImpl) b.s3SecretManager;
+ this.store = new OzoneSecretStore(b.ozoneConf,
this.s3SecretManager.getOmMetadataManager());
- isRatisEnabled = conf.getBoolean(
+ isRatisEnabled = b.ozoneConf.getBoolean(
OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY,
OMConfigKeys.OZONE_OM_RATIS_ENABLE_DEFAULT);
loadTokenSecretState(store.loadState());
+
+ }
+
+ /**
+ * Builder to help construct OzoneDelegationTokenSecretManager.
+ */
+ public static class Builder {
+ private OzoneConfiguration ozoneConf;
+ private long tokenMaxLifetime;
+ private long tokenRenewInterval;
+ private long tokenRemoverScanInterval;
+ private Text service;
+ private S3SecretManager s3SecretManager;
+ private CertificateClient certClient;
+ private String omServiceId;
+
+ public OzoneDelegationTokenSecretManager build() throws IOException {
+ return new OzoneDelegationTokenSecretManager(this);
+ }
+
+ public Builder setConf(OzoneConfiguration conf) {
+ this.ozoneConf = conf;
+ return this;
+ }
+
+ public Builder setTokenMaxLifetime(long dtMaxLifetime) {
+ this.tokenMaxLifetime = dtMaxLifetime;
+ return this;
+ }
+
+ public Builder setTokenRenewInterval(long dtRenewInterval) {
+ this.tokenRenewInterval = dtRenewInterval;
+ return this;
+ }
+
+ public Builder setTokenRemoverScanInterval(long dtRemoverScanInterval) {
+ this.tokenRemoverScanInterval = dtRemoverScanInterval;
+ return this;
+ }
+
+ public Builder setService(Text dtService) {
+ this.service = dtService;
+ return this;
+ }
+
+ public Builder setS3SecretManager(S3SecretManager s3SecManager) {
+ this.s3SecretManager = s3SecManager;
+ return this;
+ }
+
+ public Builder setCertificateClient(CertificateClient certificateClient) {
+ this.certClient = certificateClient;
+ return this;
+ }
+
+ public Builder setOmServiceId(String serviceId) {
+ this.omServiceId = serviceId;
+ return this;
+ }
}
@Override
@@ -200,6 +250,7 @@
identifier.setSequenceNumber(sequenceNum);
identifier.setMaxDate(now + getTokenMaxLifetime());
identifier.setOmCertSerialId(getOmCertificateSerialId());
+ identifier.setOmServiceId(getOmServiceId());
}
/**
@@ -213,6 +264,10 @@
return omCertificateSerialId;
}
+ private String getOmServiceId() {
+ return omServiceId;
+ }
+
/**
* Renew a delegation token.
*
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretKey.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretKey.java
similarity index 100%
rename from hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretKey.java
rename to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretKey.java
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretManager.java
similarity index 100%
rename from hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretManager.java
rename to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretManager.java
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretStore.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretStore.java
similarity index 100%
rename from hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretStore.java
rename to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretStore.java
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneNativeAuthorizer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneNativeAuthorizer.java
index 3834ab9..df98e20 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneNativeAuthorizer.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneNativeAuthorizer.java
@@ -18,16 +18,21 @@
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.annotation.InterfaceStability;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.BucketManager;
import org.apache.hadoop.ozone.om.KeyManager;
import org.apache.hadoop.ozone.om.PrefixManager;
import org.apache.hadoop.ozone.om.VolumeManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Collection;
+import java.util.Collections;
import java.util.Objects;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_REQUEST;
/**
@@ -44,17 +49,19 @@
private BucketManager bucketManager;
private KeyManager keyManager;
private PrefixManager prefixManager;
+ private Collection<String> ozAdmins;
public OzoneNativeAuthorizer() {
}
public OzoneNativeAuthorizer(VolumeManager volumeManager,
BucketManager bucketManager, KeyManager keyManager,
- PrefixManager prefixManager) {
+ PrefixManager prefixManager, Collection<String> ozoneAdmins) {
this.volumeManager = volumeManager;
this.bucketManager = bucketManager;
this.keyManager = keyManager;
this.prefixManager = prefixManager;
+ this.ozAdmins = ozoneAdmins;
}
/**
@@ -80,6 +87,15 @@
"configured to work with OzoneObjInfo type only.", INVALID_REQUEST);
}
+ // by pass all checks for admin
+ boolean isAdmin = isAdmin(context.getClientUgi());
+ if (isAdmin) {
+ return true;
+ }
+
+ boolean isListAllVolume = ((context.getAclRights() == ACLType.LIST) &&
+ objInfo.getVolumeName().equals(OzoneConsts.OZONE_ROOT));
+
// For CREATE and DELETE acl requests, the parents need to be checked
// for WRITE acl. If Key create request is received, then we need to
// check if user has WRITE acl set on Bucket and Volume. In all other cases
@@ -98,6 +114,10 @@
switch (objInfo.getResourceType()) {
case VOLUME:
LOG.trace("Checking access for volume: {}", objInfo);
+ if (isACLTypeCreate || isListAllVolume) {
+ // only admin is allowed to create volume and list all volumes
+ return false;
+ }
return volumeManager.checkAccess(objInfo, context);
case BUCKET:
LOG.trace("Checking access for bucket: {}", objInfo);
@@ -147,4 +167,26 @@
public void setPrefixManager(PrefixManager prefixManager) {
this.prefixManager = prefixManager;
}
+
+ public void setOzoneAdmins(Collection<String> ozoneAdmins) {
+ this.ozAdmins = ozoneAdmins;
+ }
+
+ public Collection<String> getOzoneAdmins() {
+ return Collections.unmodifiableCollection(this.ozAdmins);
+ }
+
+ private boolean isAdmin(UserGroupInformation callerUgi) {
+ if (ozAdmins == null) {
+ return false;
+ }
+
+ if (ozAdmins.contains(callerUgi.getShortUserName()) ||
+ ozAdmins.contains(callerUgi.getUserName()) ||
+ ozAdmins.contains(OZONE_ADMINISTRATORS_WILDCARD)) {
+ return true;
+ }
+
+ return false;
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/package-info.java
similarity index 91%
copy from hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/package-info.java
copy to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/package-info.java
index 80c1985..e5c9d4a 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/package-info.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/package-info.java
@@ -15,7 +15,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.ozone.web.ozShell;
+package org.apache.hadoop.ozone.security;
+
/**
- * Tests for ozone shell..
- */
+ * Security related classes.
+ */
\ No newline at end of file
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
deleted file mode 100644
index 6405eef..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell;
-
-import java.util.concurrent.Callable;
-
-import org.apache.hadoop.hdds.cli.GenericParentCommand;
-import org.apache.hadoop.hdds.cli.HddsVersionProvider;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import picocli.CommandLine.Command;
-import picocli.CommandLine.ParentCommand;
-
-/**
- * Common interface for command handling.
- */
-@Command(mixinStandardHelpOptions = true,
- versionProvider = HddsVersionProvider.class)
-public abstract class Handler implements Callable<Void> {
-
- protected static final Logger LOG = LoggerFactory.getLogger(Handler.class);
-
- @ParentCommand
- private GenericParentCommand parent;
-
- @Override
- public Void call() throws Exception {
- throw new UnsupportedOperationException();
- }
-
- public boolean isVerbose() {
- return parent.isVerbose();
- }
-
- public OzoneConfiguration createOzoneConfiguration() {
- return parent.createOzoneConfiguration();
- }
-
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/AddAclBucketHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/AddAclBucketHandler.java
deleted file mode 100644
index 476c7c1..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/AddAclBucketHandler.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.ozShell.bucket;
-
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.security.acl.OzoneObj;
-import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import picocli.CommandLine;
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Parameters;
-
-import java.util.Objects;
-
-import static org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType.OZONE;
-
-/**
- * Add acl handler for bucket.
- */
-@Command(name = "addacl",
- description = "Add a new ACL.")
-public class AddAclBucketHandler extends Handler {
-
- @Parameters(arity = "1..1", description = Shell.OZONE_BUCKET_URI_DESCRIPTION)
- private String uri;
-
- @CommandLine.Option(names = {"--acl", "-a"},
- required = true,
- description = "The new ACL to be added.\n" +
- "Ex: user:user1:rw or group:hadoop:rw\n" +
- "r = READ, " +
- "w = WRITE, " +
- "c = CREATE, " +
- "d = DELETE, " +
- "l = LIST, " +
- "a = ALL, " +
- "n = NONE, " +
- "x = READ_ACL, " +
- "y = WRITE_ACL.")
- private String acl;
-
- @CommandLine.Option(names = {"--store", "-s"},
- required = false,
- description = "Store type. i.e OZONE or S3")
- private String storeType;
-
- /**
- * Executes the Client Calls.
- */
- @Override
- public Void call() throws Exception {
- Objects.requireNonNull(acl,
- "You need to specify a new ACL to be added.");
- OzoneAddress address = new OzoneAddress(uri);
- address.ensureBucketAddress();
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- String volumeName = address.getVolumeName();
- String bucketName = address.getBucketName();
-
- if (isVerbose()) {
- System.out.printf("Volume Name : %s%n", volumeName);
- System.out.printf("Bucket Name : %s%n", bucketName);
- }
-
- OzoneObj obj = OzoneObjInfo.Builder.newBuilder()
- .setBucketName(bucketName)
- .setVolumeName(volumeName)
- .setResType(OzoneObj.ResourceType.BUCKET)
- .setStoreType(storeType == null ? OZONE :
- OzoneObj.StoreType.valueOf(storeType))
- .build();
-
- boolean result = client.getObjectStore().addAcl(obj,
- OzoneAcl.parseAcl(acl));
-
- String message = result
- ? ("ACL added successfully.")
- : ("ACL already exists.");
-
- System.out.println(message);
- }
- return null;
- }
-
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/CreateBucketHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/CreateBucketHandler.java
deleted file mode 100644
index 5795325..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/CreateBucketHandler.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.ozShell.bucket;
-
-import org.apache.hadoop.hdds.protocol.StorageType;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.BucketArgs;
-import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.ObjectPrinter;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Option;
-import picocli.CommandLine.Parameters;
-/**
- * create bucket handler.
- */
-@Command(name = "create",
- description = "creates a bucket in a given volume")
-public class CreateBucketHandler extends Handler {
-
- @Parameters(arity = "1..1", description = Shell.OZONE_BUCKET_URI_DESCRIPTION)
- private String uri;
-
- @Option(names = {"--bucketkey", "-k"},
- description = "bucket encryption key name")
- private String bekName;
-
- @Option(names = {"--enforcegdpr", "-g"},
- description = "if true, indicates GDPR enforced bucket, " +
- "false/unspecified indicates otherwise")
- private Boolean isGdprEnforced;
-
- /**
- * Executes create bucket.
- */
- @Override
- public Void call() throws Exception {
-
- OzoneAddress address = new OzoneAddress(uri);
- address.ensureBucketAddress();
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- String volumeName = address.getVolumeName();
- String bucketName = address.getBucketName();
-
- BucketArgs.Builder bb = new BucketArgs.Builder()
- .setStorageType(StorageType.DEFAULT)
- .setVersioning(false);
-
- if (isGdprEnforced != null) {
- if (isGdprEnforced) {
- bb.addMetadata(OzoneConsts.GDPR_FLAG, String.valueOf(Boolean.TRUE));
- } else {
- bb.addMetadata(OzoneConsts.GDPR_FLAG, String.valueOf(Boolean.FALSE));
- }
- }
-
- if (bekName != null) {
- if (!bekName.isEmpty()) {
- bb.setBucketEncryptionKey(bekName);
- } else {
- throw new IllegalArgumentException("Bucket encryption key name must" +
- " " + "be specified to enable bucket encryption!");
- }
- }
-
- if (isVerbose()) {
- System.out.printf("Volume Name : %s%n", volumeName);
- System.out.printf("Bucket Name : %s%n", bucketName);
- if (bekName != null) {
- System.out.printf("Bucket Encryption enabled with Key Name: %s%n",
- bekName);
- }
- }
-
- OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
- vol.createBucket(bucketName, bb.build());
-
- if (isVerbose()) {
- OzoneBucket bucket = vol.getBucket(bucketName);
- ObjectPrinter.printObjectAsJson(bucket);
- }
- }
- return null;
- }
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/DeleteBucketHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/DeleteBucketHandler.java
deleted file mode 100644
index 437cf05..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/DeleteBucketHandler.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.bucket;
-
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Parameters;
-
-/**
- * Delete bucket Handler.
- */
-@Command(name = "delete",
- description = "deletes an empty bucket")
-public class DeleteBucketHandler extends Handler {
-
- @Parameters(arity = "1..1", description = Shell.OZONE_BUCKET_URI_DESCRIPTION)
- private String uri;
-
- /**
- * Executes the Client Calls.
- */
- @Override
- public Void call() throws Exception {
-
- OzoneAddress address = new OzoneAddress(uri);
- address.ensureBucketAddress();
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- String volumeName = address.getVolumeName();
- String bucketName = address.getBucketName();
-
- if (isVerbose()) {
- System.out.printf("Volume Name : %s%n", volumeName);
- System.out.printf("Bucket Name : %s%n", bucketName);
- }
-
- OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
- vol.deleteBucket(bucketName);
- }
- return null;
- }
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/GetAclBucketHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/GetAclBucketHandler.java
deleted file mode 100644
index a3fc2fb..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/GetAclBucketHandler.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.ozShell.bucket;
-
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.security.acl.OzoneObj;
-import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-import picocli.CommandLine;
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Parameters;
-
-import java.util.List;
-
-import static org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType.OZONE;
-
-/**
- * Get acl handler for bucket.
- */
-@Command(name = "getacl",
- description = "List all ACLs.")
-public class GetAclBucketHandler extends Handler {
-
- @Parameters(arity = "1..1", description = Shell.OZONE_BUCKET_URI_DESCRIPTION)
- private String uri;
-
- @CommandLine.Option(names = {"--store", "-s"},
- required = false,
- description = "Store type. i.e OZONE or S3")
- private String storeType;
-
- /**
- * Executes the Client Calls.
- */
- @Override
- public Void call() throws Exception {
- OzoneAddress address = new OzoneAddress(uri);
- address.ensureBucketAddress();
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- String volumeName = address.getVolumeName();
- String bucketName = address.getBucketName();
-
- if (isVerbose()) {
- System.out.printf("Volume Name : %s%n", volumeName);
- System.out.printf("Bucket Name : %s%n", bucketName);
- }
-
- OzoneObj obj = OzoneObjInfo.Builder.newBuilder()
- .setBucketName(bucketName)
- .setVolumeName(volumeName)
- .setResType(OzoneObj.ResourceType.BUCKET)
- .setStoreType(storeType == null ? OZONE :
- OzoneObj.StoreType.valueOf(storeType))
- .build();
-
- List<OzoneAcl> result = client.getObjectStore().getAcl(obj);
-
- System.out.printf("%s%n",
- JsonUtils.toJsonStringWithDefaultPrettyPrinter(result));
- }
- return null;
- }
-
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/InfoBucketHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/InfoBucketHandler.java
deleted file mode 100644
index 1e880e0..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/InfoBucketHandler.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.ozShell.bucket;
-
-import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.ObjectPrinter;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Parameters;
-
-/**
- * Executes Info bucket.
- */
-@Command(name = "info",
- description = "returns information about a bucket")
-public class InfoBucketHandler extends Handler {
-
- @Parameters(arity = "1..1", description = Shell.OZONE_BUCKET_URI_DESCRIPTION)
- private String uri;
-
- /**
- * Executes the Client Calls.
- */
- @Override
- public Void call() throws Exception {
- OzoneAddress address = new OzoneAddress(uri);
- address.ensureBucketAddress();
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- String volumeName = address.getVolumeName();
- String bucketName = address.getBucketName();
-
- if (isVerbose()) {
- System.out.printf("Volume Name : %s%n", volumeName);
- System.out.printf("Bucket Name : %s%n", bucketName);
- }
-
- OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
- OzoneBucket bucket = vol.getBucket(bucketName);
-
- ObjectPrinter.printObjectAsJson(bucket);
- }
-
- return null;
- }
-
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java
deleted file mode 100644
index bd05d80..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.bucket;
-
-import java.util.Iterator;
-
-import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.ObjectPrinter;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Help.Visibility;
-import picocli.CommandLine.Option;
-import picocli.CommandLine.Parameters;
-
-/**
- * Executes List Bucket.
- */
-@Command(name = "list",
- aliases = "ls",
- description = "lists the buckets in a volume.")
-public class ListBucketHandler extends Handler {
-
- @Parameters(arity = "1..1", description = Shell.OZONE_VOLUME_URI_DESCRIPTION)
- private String uri;
-
- @Option(names = {"--length", "-l"},
- description = "Limit of the max results",
- defaultValue = "100",
- showDefaultValue = Visibility.ALWAYS)
- private int maxBuckets;
-
- @Option(names = {"--start", "-s"},
- description = "The bucket to start the listing from.\n" +
- "This will be excluded from the result.")
- private String startBucket;
-
- @Option(names = {"--prefix", "-p"},
- description = "Prefix to filter the buckets")
- private String prefix;
- /**
- * Executes the Client Calls.
- */
- @Override
- public Void call() throws Exception {
-
- OzoneAddress address = new OzoneAddress(uri);
- address.ensureVolumeAddress();
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- String volumeName = address.getVolumeName();
- if (maxBuckets < 1) {
- throw new IllegalArgumentException(
- "the length should be a positive number");
- }
-
- if (isVerbose()) {
- System.out.printf("Volume Name : %s%n", volumeName);
- }
-
-
- OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
- Iterator<? extends OzoneBucket> bucketIterator =
- vol.listBuckets(prefix, startBucket);
-
- int counter = 0;
- while (maxBuckets > 0 && bucketIterator.hasNext()) {
- ObjectPrinter.printObjectAsJson(bucketIterator.next());
-
- maxBuckets -= 1;
- counter++;
- }
-
- if (isVerbose()) {
- System.out.printf("Found : %d buckets for volume : %s ",
- counter, volumeName);
- }
- }
-
- return null;
- }
-
-}
-
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/RemoveAclBucketHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/RemoveAclBucketHandler.java
deleted file mode 100644
index 6359a77..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/RemoveAclBucketHandler.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.ozShell.bucket;
-
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.security.acl.OzoneObj;
-import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import picocli.CommandLine;
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Parameters;
-
-import java.util.Objects;
-
-import static org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType.OZONE;
-
-/**
- * Executes Info bucket.
- */
-@Command(name = "removeacl",
- description = "Remove an existing ACL.")
-public class RemoveAclBucketHandler extends Handler {
-
- @Parameters(arity = "1..1", description = Shell.OZONE_BUCKET_URI_DESCRIPTION)
- private String uri;
-
- @CommandLine.Option(names = {"--acl", "-a"},
- required = true,
- description = "The ACL to be removed.\n" +
- "Ex: user:user1:rw or group:hadoop:rw\n" +
- "r = READ, " +
- "w = WRITE, " +
- "c = CREATE, " +
- "d = DELETE, " +
- "l = LIST, " +
- "a = ALL, " +
- "n = NONE, " +
- "x = READ_ACL, " +
- "y = WRITE_ACL.")
- private String acl;
-
- @CommandLine.Option(names = {"--store", "-s"},
- required = false,
- description = "Store type. i.e OZONE or S3")
- private String storeType;
-
- /**
- * Remove acl handler for bucket.
- */
- @Override
- public Void call() throws Exception {
- Objects.requireNonNull(acl,
- "You need to specify an ACL to be removed.");
- OzoneAddress address = new OzoneAddress(uri);
- address.ensureBucketAddress();
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- String volumeName = address.getVolumeName();
- String bucketName = address.getBucketName();
-
- if (isVerbose()) {
- System.out.printf("Volume Name : %s%n", volumeName);
- System.out.printf("Bucket Name : %s%n", bucketName);
- }
-
- OzoneObj obj = OzoneObjInfo.Builder.newBuilder()
- .setBucketName(bucketName)
- .setVolumeName(volumeName)
- .setResType(OzoneObj.ResourceType.BUCKET)
- .setStoreType(storeType == null ? OZONE :
- OzoneObj.StoreType.valueOf(storeType))
- .build();
-
- boolean result = client.getObjectStore().removeAcl(obj,
- OzoneAcl.parseAcl(acl));
-
- String message = result
- ? ("ACL removed successfully.")
- : ("ACL doesn't exist.");
-
- System.out.println(message);
- }
-
- return null;
- }
-
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/SetAclBucketHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/SetAclBucketHandler.java
deleted file mode 100644
index eae1661..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/SetAclBucketHandler.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.ozShell.bucket;
-
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.security.acl.OzoneObj;
-import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import picocli.CommandLine;
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Parameters;
-
-import java.util.Objects;
-
-import static org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType.OZONE;
-
-/**
- * Set acl handler for bucket.
- */
-@Command(name = "setacl",
- description = "Set one or more ACLs, replacing the existing ones.")
-public class SetAclBucketHandler extends Handler {
-
- @Parameters(arity = "1..1", description = Shell.OZONE_BUCKET_URI_DESCRIPTION)
- private String uri;
-
- @CommandLine.Option(names = {"--acls", "-al"},
- required = true,
- description = "A comma separated list of ACLs to be set.\n" +
- "Ex: user:user1:rw,user:user2:a,group:hadoop:a\n" +
- "r = READ, " +
- "w = WRITE, " +
- "c = CREATE, " +
- "d = DELETE, " +
- "l = LIST, " +
- "a = ALL, " +
- "n = NONE, " +
- "x = READ_ACL, " +
- "y = WRITE_ACL.")
- private String acls;
-
- @CommandLine.Option(names = {"--store", "-s"},
- required = false,
- description = "Store type. i.e OZONE or S3")
- private String storeType;
-
- /**
- * Executes the Client Calls.
- */
- @Override
- public Void call() throws Exception {
- Objects.requireNonNull(acls,
- "You need to specify one or more ACLs to be set.");
- OzoneAddress address = new OzoneAddress(uri);
- address.ensureBucketAddress();
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- String volumeName = address.getVolumeName();
- String bucketName = address.getBucketName();
-
- if (isVerbose()) {
- System.out.printf("Volume Name : %s%n", volumeName);
- System.out.printf("Bucket Name : %s%n", bucketName);
- }
-
- OzoneObj obj = OzoneObjInfo.Builder.newBuilder()
- .setBucketName(bucketName)
- .setVolumeName(volumeName)
- .setResType(OzoneObj.ResourceType.BUCKET)
- .setStoreType(storeType == null ? OZONE :
- OzoneObj.StoreType.valueOf(storeType))
- .build();
-
- client.getObjectStore().setAcl(obj, OzoneAcl.parseAcls(acls));
- System.out.println("ACL(s) set successfully.");
- }
-
- return null;
- }
-
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/AddAclKeyHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/AddAclKeyHandler.java
deleted file mode 100644
index 40c2b47..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/AddAclKeyHandler.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.ozShell.keys;
-
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.security.acl.OzoneObj;
-import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import picocli.CommandLine;
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Parameters;
-
-import java.util.Objects;
-
-import static org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType.OZONE;
-
-/**
- * Add acl handler for key.
- */
-@Command(name = "addacl",
- description = "Add a new ACL.")
-public class AddAclKeyHandler extends Handler {
-
- @Parameters(arity = "1..1", description = Shell.OZONE_BUCKET_URI_DESCRIPTION)
- private String uri;
-
- @CommandLine.Option(names = {"--acl", "-a"},
- required = true,
- description = "The new ACL to be added.\n" +
- "Ex: user:user1:rw or group:hadoop:rw\n" +
- "r = READ, " +
- "w = WRITE, " +
- "c = CREATE, " +
- "d = DELETE, " +
- "l = LIST, " +
- "a = ALL, " +
- "n = NONE, " +
- "x = READ_ACL, " +
- "y = WRITE_ACL.")
- private String acl;
-
- @CommandLine.Option(names = {"--store", "-s"},
- required = false,
- description = "Store type. i.e OZONE or S3")
- private String storeType;
-
- /**
- * Executes the Client Calls.
- */
- @Override
- public Void call() throws Exception {
- Objects.requireNonNull(acl,
- "You need to specify a new ACL to be added.");
- OzoneAddress address = new OzoneAddress(uri);
- address.ensureKeyAddress();
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- String volumeName = address.getVolumeName();
- String bucketName = address.getBucketName();
- String keyName = address.getKeyName();
-
- if (isVerbose()) {
- System.out.printf("Volume Name : %s%n", volumeName);
- System.out.printf("Bucket Name : %s%n", bucketName);
- System.out.printf("Key Name : %s%n", keyName);
- }
-
- OzoneObj obj = OzoneObjInfo.Builder.newBuilder()
- .setBucketName(bucketName)
- .setVolumeName(volumeName)
- .setKeyName(address.getKeyName())
- .setResType(OzoneObj.ResourceType.KEY)
- .setStoreType(storeType == null ? OZONE :
- OzoneObj.StoreType.valueOf(storeType))
- .build();
-
- boolean result = client.getObjectStore().addAcl(obj,
- OzoneAcl.parseAcl(acl));
-
- String message = result
- ? ("ACL added successfully.")
- : ("ACL already exists.");
-
- System.out.println(message);
- }
-
- return null;
- }
-
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/DeleteKeyHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/DeleteKeyHandler.java
deleted file mode 100644
index be9ab99..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/DeleteKeyHandler.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.keys;
-
-import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Parameters;
-
-/**
- * Executes Delete Key.
- */
-@Command(name = "delete",
- description = "deletes an existing key")
-public class DeleteKeyHandler extends Handler {
-
- @Parameters(arity = "1..1", description = Shell.OZONE_KEY_URI_DESCRIPTION)
- private String uri;
-
- /**
- * Executes the Client Calls.
- */
- @Override
- public Void call() throws Exception {
-
- OzoneAddress address = new OzoneAddress(uri);
- address.ensureKeyAddress();
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- String volumeName = address.getVolumeName();
- String bucketName = address.getBucketName();
- String keyName = address.getKeyName();
-
-
- if (isVerbose()) {
- System.out.printf("Volume Name : %s%n", volumeName);
- System.out.printf("Bucket Name : %s%n", bucketName);
- System.out.printf("Key Name : %s%n", keyName);
- }
-
- OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
- OzoneBucket bucket = vol.getBucket(bucketName);
- bucket.deleteKey(keyName);
- }
-
- return null;
- }
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetAclKeyHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetAclKeyHandler.java
deleted file mode 100644
index bef6773..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetAclKeyHandler.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.ozShell.keys;
-
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.security.acl.OzoneObj;
-import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-import picocli.CommandLine;
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Parameters;
-
-import java.util.List;
-
-import static org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType.OZONE;
-
-/**
- * Get acl handler for Key.
- */
-@Command(name = "getacl",
- description = "List all ACLs.")
-public class GetAclKeyHandler extends Handler {
-
- @Parameters(arity = "1..1", description = Shell.OZONE_BUCKET_URI_DESCRIPTION)
- private String uri;
-
- @CommandLine.Option(names = {"--store", "-s"},
- required = false,
- description = "Store type. i.e OZONE or S3")
- private String storeType;
-
- /**
- * Executes the Client Calls.
- */
- @Override
- public Void call() throws Exception {
- OzoneAddress address = new OzoneAddress(uri);
- address.ensureKeyAddress();
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- String volumeName = address.getVolumeName();
- String bucketName = address.getBucketName();
- String keyName = address.getKeyName();
-
- if (isVerbose()) {
- System.out.printf("Volume Name : %s%n", volumeName);
- System.out.printf("Bucket Name : %s%n", bucketName);
- System.out.printf("Key Name : %s%n", keyName);
- }
-
- OzoneObj obj = OzoneObjInfo.Builder.newBuilder()
- .setBucketName(bucketName)
- .setVolumeName(volumeName)
- .setKeyName(keyName)
- .setResType(OzoneObj.ResourceType.KEY)
- .setStoreType(storeType == null ? OZONE :
- OzoneObj.StoreType.valueOf(storeType))
- .build();
-
- List<OzoneAcl> result = client.getObjectStore().getAcl(obj);
-
- System.out.printf("%s%n",
- JsonUtils.toJsonStringWithDefaultPrettyPrinter(result));
- }
-
- return null;
- }
-
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
deleted file mode 100644
index d9f632b..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.keys;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.InputStream;
-import java.io.OutputStream;
-
-import org.apache.hadoop.conf.StorageUnit;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneClientException;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-
-import org.apache.commons.codec.digest.DigestUtils;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY;
-
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Parameters;
-
-/**
- * Gets an existing key.
- */
-@Command(name = "get",
- description = "Gets a specific key from ozone server")
-public class GetKeyHandler extends Handler {
-
- @Parameters(index = "0", arity = "1..1", description =
- Shell.OZONE_KEY_URI_DESCRIPTION)
- private String uri;
-
- @Parameters(index = "1", arity = "1..1",
- description = "File path to download the key to")
- private String fileName;
-
- /**
- * Executes the Client Calls.
- */
- @Override
- public Void call() throws Exception {
-
- OzoneAddress address = new OzoneAddress(uri);
- address.ensureKeyAddress();
-
- OzoneConfiguration conf = createOzoneConfiguration();
-
- try (OzoneClient client = address.createClient(conf)) {
-
- String volumeName = address.getVolumeName();
- String bucketName = address.getBucketName();
- String keyName = address.getKeyName();
-
- if (isVerbose()) {
- System.out.printf("Volume Name : %s%n", volumeName);
- System.out.printf("Bucket Name : %s%n", bucketName);
- System.out.printf("Key Name : %s%n", keyName);
- }
-
- File dataFile = new File(fileName);
-
- if (dataFile.exists() && dataFile.isDirectory()) {
- dataFile = new File(fileName, keyName);
- }
-
- if (dataFile.exists()) {
- throw new OzoneClientException(dataFile.getPath() + " exists."
- + " Download would overwrite an existing file. Aborting.");
- }
-
- int chunkSize = (int) conf.getStorageSize(OZONE_SCM_CHUNK_SIZE_KEY,
- OZONE_SCM_CHUNK_SIZE_DEFAULT, StorageUnit.BYTES);
-
- OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
- OzoneBucket bucket = vol.getBucket(bucketName);
- try (InputStream input = bucket.readKey(keyName);
- OutputStream output = new FileOutputStream(dataFile)) {
- IOUtils.copyBytes(input, output, chunkSize);
- }
-
- if (isVerbose()) {
- try (InputStream stream = new FileInputStream(dataFile)) {
- String hash = DigestUtils.md5Hex(stream);
- System.out.printf("Downloaded file hash : %s%n", hash);
- }
- }
- }
- return null;
- }
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/InfoKeyHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/InfoKeyHandler.java
deleted file mode 100644
index 54934f5..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/InfoKeyHandler.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.keys;
-
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneKeyDetails;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.ObjectPrinter;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Parameters;
-
-/**
- * Executes Info Object.
- */
-@Command(name = "info",
- description = "returns information about an existing key")
-public class InfoKeyHandler extends Handler {
-
- @Parameters(arity = "1..1", description = Shell.OZONE_KEY_URI_DESCRIPTION)
- private String uri;
- /**
- * Executes the Client Calls.
- */
- @Override
- public Void call() throws Exception {
-
- OzoneAddress address = new OzoneAddress(uri);
- address.ensureKeyAddress();
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- String volumeName = address.getVolumeName();
- String bucketName = address.getBucketName();
- String keyName = address.getKeyName();
-
- if (isVerbose()) {
- System.out.printf("Volume Name : %s%n", volumeName);
- System.out.printf("Bucket Name : %s%n", bucketName);
- System.out.printf("Key Name : %s%n", keyName);
- }
-
- OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
- OzoneBucket bucket = vol.getBucket(bucketName);
- OzoneKeyDetails key = bucket.getKey(keyName);
- // For compliance/security, GDPR Secret & Algorithm details are removed
- // from local copy of metadata before printing. This doesn't remove these
- // from Ozone Manager's actual metadata.
- key.getMetadata().remove(OzoneConsts.GDPR_SECRET);
- key.getMetadata().remove(OzoneConsts.GDPR_ALGORITHM);
-
- ObjectPrinter.printObjectAsJson(key);
- }
-
- return null;
- }
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java
deleted file mode 100644
index 8d699d1..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.keys;
-
-import java.util.Iterator;
-
-import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneKey;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.ObjectPrinter;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Option;
-import picocli.CommandLine.Parameters;
-
-/**
- * Executes List Keys.
- */
-@Command(name = "list",
- aliases = "ls",
- description = "list all keys in a given bucket")
-public class ListKeyHandler extends Handler {
-
- @Parameters(arity = "1..1", description = Shell.OZONE_BUCKET_URI_DESCRIPTION)
- private String uri;
-
- @Option(names = {"--length", "-l"},
- description = "Limit of the max results",
- defaultValue = "100")
- private int maxKeys;
-
- @Option(names = {"--start", "-s"},
- description = "The key to start the listing from.\n" +
- "This will be excluded from the result.")
- private String startKey;
-
- @Option(names = {"--prefix", "-p"},
- description = "Prefix to filter the key")
- private String prefix;
-
- /**
- * Executes the Client Calls.
- */
- @Override
- public Void call() throws Exception {
-
- OzoneAddress address = new OzoneAddress(uri);
- address.ensureBucketAddress();
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- String volumeName = address.getVolumeName();
- String bucketName = address.getBucketName();
-
- if (maxKeys < 1) {
- throw new IllegalArgumentException(
- "the length should be a positive number");
- }
-
- if (isVerbose()) {
- System.out.printf("Volume Name : %s%n", volumeName);
- System.out.printf("bucket Name : %s%n", bucketName);
- }
-
- OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
- OzoneBucket bucket = vol.getBucket(bucketName);
- Iterator<? extends OzoneKey> keyIterator = bucket.listKeys(prefix,
- startKey);
-
- int maxKeyLimit = maxKeys;
-
- int counter = 0;
- while (maxKeys > 0 && keyIterator.hasNext()) {
- OzoneKey ozoneKey = keyIterator.next();
- ObjectPrinter.printObjectAsJson(ozoneKey);
- maxKeys -= 1;
- counter++;
- }
-
- // More keys were returned notify about max length
- if (keyIterator.hasNext()) {
- System.out.println("Listing first " + maxKeyLimit + " entries of the " +
- "result. Use --length (-l) to override max returned keys.");
- } else if (isVerbose()) {
- System.out.printf("Found : %d keys for bucket %s in volume : %s ",
- counter, bucketName, volumeName);
- }
- }
-
- return null;
- }
-
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
deleted file mode 100644
index 7bde7f8..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.keys;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.StorageUnit;
-import org.apache.hadoop.hdds.client.ReplicationFactor;
-import org.apache.hadoop.hdds.client.ReplicationType;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-
-import org.apache.commons.codec.digest.DigestUtils;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT;
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Option;
-import picocli.CommandLine.Parameters;
-
-/**
- * Puts a file into an ozone bucket.
- */
-@Command(name = "put",
- description = "creates or overwrites an existing key")
-public class PutKeyHandler extends Handler {
-
- @Parameters(index = "0", arity = "1..1", description =
- Shell.OZONE_KEY_URI_DESCRIPTION)
- private String uri;
-
- @Parameters(index = "1", arity = "1..1", description = "File to upload")
- private String fileName;
-
- @Option(names = {"-r", "--replication"},
- description = "Replication factor of the new key. (use ONE or THREE) "
- + "Default is specified in the cluster-wide config.")
- private ReplicationFactor replicationFactor;
- /**
- * Executes the Client Calls.
- */
- @Override
- public Void call() throws Exception {
- OzoneAddress address = new OzoneAddress(uri);
- address.ensureKeyAddress();
-
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- String volumeName = address.getVolumeName();
- String bucketName = address.getBucketName();
- String keyName = address.getKeyName();
-
- if (isVerbose()) {
- System.out.printf("Volume Name : %s%n", volumeName);
- System.out.printf("Bucket Name : %s%n", bucketName);
- System.out.printf("Key Name : %s%n", keyName);
- }
-
- File dataFile = new File(fileName);
-
- if (isVerbose()) {
- try (InputStream stream = new FileInputStream(dataFile)) {
- String hash = DigestUtils.md5Hex(stream);
- System.out.printf("File Hash : %s%n", hash);
- }
- }
-
- Configuration conf = new OzoneConfiguration();
- if (replicationFactor == null) {
- replicationFactor = ReplicationFactor.valueOf(
- conf.getInt(OZONE_REPLICATION, OZONE_REPLICATION_DEFAULT));
- }
-
- ReplicationType replicationType = ReplicationType.valueOf(
- conf.get(OZONE_REPLICATION_TYPE, OZONE_REPLICATION_TYPE_DEFAULT));
- OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
- OzoneBucket bucket = vol.getBucket(bucketName);
-
- Map<String, String> keyMetadata = new HashMap<>();
- String gdprEnabled = bucket.getMetadata().get(OzoneConsts.GDPR_FLAG);
- if (Boolean.parseBoolean(gdprEnabled)) {
- keyMetadata.put(OzoneConsts.GDPR_FLAG, Boolean.TRUE.toString());
- }
-
- int chunkSize = (int) conf.getStorageSize(OZONE_SCM_CHUNK_SIZE_KEY,
- OZONE_SCM_CHUNK_SIZE_DEFAULT, StorageUnit.BYTES);
- try (InputStream input = new FileInputStream(dataFile);
- OutputStream output = bucket.createKey(keyName, dataFile.length(),
- replicationType, replicationFactor, keyMetadata)) {
- IOUtils.copyBytes(input, output, chunkSize);
- }
- }
- return null;
- }
-
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/RemoveAclKeyHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/RemoveAclKeyHandler.java
deleted file mode 100644
index ec419cb..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/RemoveAclKeyHandler.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.ozShell.keys;
-
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.security.acl.OzoneObj;
-import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import picocli.CommandLine;
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Parameters;
-
-import java.util.Objects;
-
-import static org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType.OZONE;
-
-/**
- * Remove acl handler for key.
- */
-@Command(name = "removeacl",
- description = "Remove an existing ACL.")
-public class RemoveAclKeyHandler extends Handler {
-
- @Parameters(arity = "1..1", description = Shell.OZONE_BUCKET_URI_DESCRIPTION)
- private String uri;
-
- @CommandLine.Option(names = {"--acl", "-a"},
- required = true,
- description = "The ACL to be removed.\n" +
- "Ex: user:user1:rw or group:hadoop:rw\n" +
- "r = READ, " +
- "w = WRITE, " +
- "c = CREATE, " +
- "d = DELETE, " +
- "l = LIST, " +
- "a = ALL, " +
- "n = NONE, " +
- "x = READ_ACL, " +
- "y = WRITE_ACL.")
- private String acl;
-
- @CommandLine.Option(names = {"--store", "-s"},
- required = false,
- description = "Store type. i.e OZONE or S3")
- private String storeType;
-
- /**
- * Executes the Client Calls.
- */
- @Override
- public Void call() throws Exception {
- Objects.requireNonNull(acl,
- "You need to specify an ACL to be removed.");
- OzoneAddress address = new OzoneAddress(uri);
- address.ensureKeyAddress();
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- String volumeName = address.getVolumeName();
- String bucketName = address.getBucketName();
- String keyName = address.getKeyName();
-
- if (isVerbose()) {
- System.out.printf("Volume Name : %s%n", volumeName);
- System.out.printf("Bucket Name : %s%n", bucketName);
- System.out.printf("Key Name : %s%n", keyName);
- }
-
- OzoneObj obj = OzoneObjInfo.Builder.newBuilder()
- .setBucketName(bucketName)
- .setVolumeName(volumeName)
- .setKeyName(keyName)
- .setResType(OzoneObj.ResourceType.KEY)
- .setStoreType(storeType == null ? OZONE :
- OzoneObj.StoreType.valueOf(storeType))
- .build();
-
- boolean result = client.getObjectStore().removeAcl(obj,
- OzoneAcl.parseAcl(acl));
-
- String message = result
- ? ("ACL removed successfully.")
- : ("ACL doesn't exist.");
-
- System.out.println(message);
- }
-
- return null;
- }
-
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/RenameKeyHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/RenameKeyHandler.java
deleted file mode 100644
index 3231372..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/RenameKeyHandler.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.ozShell.keys;
-
-import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Parameters;
-
-/**
- * Renames an existing key.
- */
-@Command(name = "rename",
- description = "renames an existing key")
-public class RenameKeyHandler extends Handler {
-
- @Parameters(index = "0", arity = "1..1",
- description = Shell.OZONE_BUCKET_URI_DESCRIPTION)
- private String uri;
-
- @Parameters(index = "1", arity = "1..1",
- description = "The existing key to be renamed")
- private String fromKey;
-
- @Parameters(index = "2", arity = "1..1",
- description = "The new desired name of the key")
- private String toKey;
-
- @Override
- public Void call() throws Exception {
- OzoneAddress address = new OzoneAddress(uri);
- address.ensureBucketAddress();
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- String volumeName = address.getVolumeName();
- String bucketName = address.getBucketName();
-
- if (isVerbose()) {
- System.out.printf("Volume Name : %s%n", volumeName);
- System.out.printf("Bucket Name : %s%n", bucketName);
- }
-
- OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
- OzoneBucket bucket = vol.getBucket(bucketName);
- bucket.renameKey(fromKey, toKey);
-
- if (isVerbose()) {
- System.out.printf("Renamed Key : %s to %s%n", fromKey, toKey);
- }
- }
-
- return null;
- }
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/SetAclKeyHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/SetAclKeyHandler.java
deleted file mode 100644
index 80d77d1..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/SetAclKeyHandler.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.ozShell.keys;
-
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.security.acl.OzoneObj;
-import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import picocli.CommandLine;
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Parameters;
-
-import java.util.Objects;
-
-import static org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType.OZONE;
-
-/**
- * Set acl handler for Key.
- */
-@Command(name = "setacl",
- description = "Set one or more ACLs, replacing the existing ones.")
-public class SetAclKeyHandler extends Handler {
-
- @Parameters(arity = "1..1", description = Shell.OZONE_BUCKET_URI_DESCRIPTION)
- private String uri;
-
- @CommandLine.Option(names = {"--acls", "-al"},
- required = true,
- description = "A comma separated list of ACLs to be set.\n" +
- "Ex: user:user1:rw,user:user2:a,group:hadoop:a\n" +
- "r = READ, " +
- "w = WRITE, " +
- "c = CREATE, " +
- "d = DELETE, " +
- "l = LIST, " +
- "a = ALL, " +
- "n = NONE, " +
- "x = READ_ACL, " +
- "y = WRITE_ACL.")
- private String acls;
-
- @CommandLine.Option(names = {"--store", "-s"},
- required = false,
- description = "Store type. i.e OZONE or S3")
- private String storeType;
-
- /**
- * Executes the Client Calls.
- */
- @Override
- public Void call() throws Exception {
- Objects.requireNonNull(acls,
- "You need to specify one or more ACLs to be set.");
- OzoneAddress address = new OzoneAddress(uri);
- address.ensureKeyAddress();
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- String volumeName = address.getVolumeName();
- String bucketName = address.getBucketName();
- String keyName = address.getKeyName();
-
- if (isVerbose()) {
- System.out.printf("Volume Name : %s%n", volumeName);
- System.out.printf("Bucket Name : %s%n", bucketName);
- }
-
- OzoneObj obj = OzoneObjInfo.Builder.newBuilder()
- .setBucketName(bucketName)
- .setVolumeName(volumeName)
- .setKeyName(keyName)
- .setResType(OzoneObj.ResourceType.KEY)
- .setStoreType(storeType == null ? OZONE :
- OzoneObj.StoreType.valueOf(storeType))
- .build();
-
- client.getObjectStore().setAcl(obj, OzoneAcl.parseAcls(acls));
- System.out.println("ACL(s) set successfully.");
- }
-
- return null;
- }
-
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/package-info.java
deleted file mode 100644
index 1deb7ad..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * Netty-based HTTP server implementation for Ozone.
- */
-package org.apache.hadoop.ozone.web.ozShell.keys;
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/package-info.java
deleted file mode 100644
index e33b6e7..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/package-info.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * ozShell Class acts as the command line interface to
- * the ozone Rest Client.
- */
-package org.apache.hadoop.ozone.web.ozShell;
-
-/**
- A simple CLI to work against Ozone.
- **/
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/s3/GetS3SecretHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/s3/GetS3SecretHandler.java
deleted file mode 100644
index 52bd817..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/s3/GetS3SecretHandler.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.ozShell.s3;
-
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.security.UserGroupInformation;
-import picocli.CommandLine.Command;
-
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
-
-/**
- * Executes getsecret calls.
- */
-@Command(name = "getsecret",
- description = "Returns s3 secret for current user")
-public class GetS3SecretHandler extends S3Handler {
-
- public static final String OZONE_GETS3SECRET_ERROR = "This command is not" +
- " supported in unsecure clusters.";
-
- /**
- * Executes getS3Secret.
- */
- @Override
- public Void call() throws Exception {
- OzoneConfiguration ozoneConfiguration = createOzoneConfiguration();
- try (OzoneClient client =
- new OzoneAddress().createClientForS3Commands(ozoneConfiguration,
- getOmServiceID())) {
-
- // getS3Secret works only with secured clusters
- if (ozoneConfiguration.getBoolean(OZONE_SECURITY_ENABLED_KEY, false)) {
- System.out.println(
- client.getObjectStore().getS3Secret(
- UserGroupInformation.getCurrentUser().getUserName()
- ).toString()
- );
- } else {
- // log a warning message for unsecured cluster
- System.out.println(OZONE_GETS3SECRET_ERROR);
- }
- }
-
- return null;
- }
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/s3/S3BucketMapping.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/s3/S3BucketMapping.java
deleted file mode 100644
index ae61523..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/s3/S3BucketMapping.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.ozShell.s3;
-
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Parameters;
-
-/**
- * S3Bucket mapping handler, which returns volume name and Ozone fs uri for
- * that bucket.
- */
-@Command(name = "path",
- description = "Returns the ozone path for S3Bucket")
-public class S3BucketMapping extends S3Handler {
-
- @Parameters(arity = "1..1", description = "Name of the s3 bucket.")
- private String s3BucketName;
-
- /**
- * Executes create bucket.
- */
- @Override
- public Void call() throws Exception {
-
- OzoneAddress ozoneAddress = new OzoneAddress();
- try (OzoneClient client =
- ozoneAddress.createClientForS3Commands(
- createOzoneConfiguration(), getOmServiceID())) {
-
- String mapping =
- client.getObjectStore().getOzoneBucketMapping(s3BucketName);
- String volumeName =
- client.getObjectStore().getOzoneVolumeName(s3BucketName);
-
- if (isVerbose()) {
- System.out.printf("Mapping created for S3Bucket is : %s%n", mapping);
- }
-
- System.out.printf("Volume name for S3Bucket is : %s%n", volumeName);
-
- String ozoneFsUri = String.format("%s://%s.%s", OzoneConsts
- .OZONE_URI_SCHEME, s3BucketName, volumeName);
-
- System.out.printf("Ozone FileSystem Uri is : %s%n", ozoneFsUri);
- }
-
- return null;
- }
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/token/CancelTokenHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/token/CancelTokenHandler.java
deleted file mode 100644
index cd4496d..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/token/CancelTokenHandler.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.token;
-
-import org.apache.hadoop.ozone.OzoneSecurityUtil;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.security.token.Token;
-import picocli.CommandLine;
-import picocli.CommandLine.Command;
-
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-
-/**
- * Executes cancelDelegationToken api.
- */
-@Command(name = "cancel",
- description = "cancel a delegation token.")
-public class CancelTokenHandler extends Handler {
-
- @CommandLine.Option(names = {"--token", "-t"},
- description = "file containing encoded token",
- defaultValue = "/tmp/token.txt",
- showDefaultValue = CommandLine.Help.Visibility.ALWAYS)
- private String tokenFile;
-
- /**
- * Executes the Client Calls.
- */
- @Override
- public Void call() throws Exception {
- OzoneAddress address = new OzoneAddress("");
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- if (!OzoneSecurityUtil.isSecurityEnabled(createOzoneConfiguration())) {
- System.err.println("Error:Token operations work only when security is" +
- " " + "enabled. To enable security set ozone.security.enabled to " +
- "true.");
- return null;
- }
-
- if (Files.notExists(Paths.get(tokenFile))) {
- System.err.println("Error:Cancel token operation failed as token file: "
- + tokenFile + " containing encoded token doesn't exist.");
- return null;
- }
- Token token = new Token();
- token.decodeFromUrlString(
- new String(Files.readAllBytes(Paths.get(tokenFile)),
- StandardCharsets.UTF_8));
- client.getObjectStore().cancelDelegationToken(token);
- }
-
- return null;
- }
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/token/GetTokenHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/token/GetTokenHandler.java
deleted file mode 100644
index 467a9dc..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/token/GetTokenHandler.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.token;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ozone.OzoneSecurityUtil;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import picocli.CommandLine;
-import picocli.CommandLine.Command;
-
-import java.util.Objects;
-
-/**
- * Executes getDelegationToken api.
- */
-@Command(name = "get",
- description = "get a delegation token.")
-public class GetTokenHandler extends Handler {
-
-
-
- @CommandLine.Option(names = {"--renewer", "-r"},
- description = "Token renewer",
- showDefaultValue = CommandLine.Help.Visibility.ALWAYS)
- private String renewer;
-
- /**
- * Executes the Client Calls.
- */
- @Override
- public Void call() throws Exception {
- OzoneAddress address = new OzoneAddress();
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- if (!OzoneSecurityUtil.isSecurityEnabled(createOzoneConfiguration())) {
- System.err.println("Error:Token operations work only when security is" +
- " " + "enabled. To enable security set ozone.security.enabled to " +
- "true.");
- return null;
- }
-
- if (StringUtils.isEmpty(renewer)) {
- renewer = UserGroupInformation.getCurrentUser().getShortUserName();
- }
- Token token =
- client.getObjectStore().getDelegationToken(new Text(renewer));
- if (Objects.isNull(token)) {
- System.err.println("Error: Get delegation token operation failed. " +
- "Check" + " OzoneManager logs for more details.");
- return null;
- }
-
- System.out.printf("%s", JsonUtils.toJsonStringWithDefaultPrettyPrinter(
- token.encodeToUrlString()));
- }
- return null;
- }
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/token/PrintTokenHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/token/PrintTokenHandler.java
deleted file mode 100644
index 24f9100..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/token/PrintTokenHandler.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.token;
-
-import org.apache.hadoop.ozone.OzoneSecurityUtil;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-import org.apache.hadoop.security.token.Token;
-import picocli.CommandLine;
-import picocli.CommandLine.Command;
-
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-
-/**
- * Executes getDelegationToken api.
- */
-@Command(name = "print",
- description = "print a delegation token.")
-public class PrintTokenHandler extends Handler {
-
- @CommandLine.Option(names = {"--token", "-t"},
- description = "file containing encoded token",
- defaultValue = "/tmp/token.txt",
- showDefaultValue = CommandLine.Help.Visibility.ALWAYS)
- private String tokenFile;
-
- /**
- * Executes the Client Calls.
- */
- @Override
- public Void call() throws Exception {
- if (!OzoneSecurityUtil.isSecurityEnabled(createOzoneConfiguration())) {
- System.err.println("Error:Token operations work only when security is " +
- "enabled. To enable security set ozone.security.enabled to true.");
- return null;
- }
-
- if (Files.notExists(Paths.get(tokenFile))) {
- System.err.println("Error: Print token operation failed as token file: "
- + tokenFile + " containing encoded token doesn't exist.");
- return null;
- }
-
- String encodedToken = new String(Files.readAllBytes(Paths.get(tokenFile)),
- StandardCharsets.UTF_8);
- Token token = new Token();
- token.decodeFromUrlString(encodedToken);
-
- System.out.printf("%s", JsonUtils.toJsonStringWithDefaultPrettyPrinter(
- token.toString()));
- return null;
- }
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/token/RenewTokenHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/token/RenewTokenHandler.java
deleted file mode 100644
index dc32cd3..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/token/RenewTokenHandler.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.token;
-
-import org.apache.hadoop.ozone.OzoneSecurityUtil;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.security.token.Token;
-import picocli.CommandLine;
-import picocli.CommandLine.Command;
-
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-
-/**
- * Executes renewDelegationToken api.
- */
-@Command(name = "renew",
- description = "renew a delegation token.")
-public class RenewTokenHandler extends Handler {
-
- @CommandLine.Option(names = {"--token", "-t"},
- description = "file containing encoded token",
- defaultValue = "/tmp/token.txt",
- showDefaultValue = CommandLine.Help.Visibility.ALWAYS)
- private String tokenFile;
-
- /**
- * Executes the Client Calls.
- */
- @Override
- public Void call() throws Exception {
- OzoneAddress address = new OzoneAddress("");
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- if (!OzoneSecurityUtil.isSecurityEnabled(createOzoneConfiguration())) {
- System.err.println("Error:Token operations work only when security is" +
- " " + "enabled. To enable security set ozone.security.enabled to " +
- "true.");
- return null;
- }
-
- if (Files.notExists(Paths.get(tokenFile))) {
- System.err.println("Error:Renew token operation failed as token file: "
- + tokenFile + " containing encoded token doesn't exist.");
- return null;
- }
- Token token = new Token();
- token.decodeFromUrlString(
- new String(Files.readAllBytes(Paths.get(tokenFile)),
- StandardCharsets.UTF_8));
- long expiryTime = client.getObjectStore().renewDelegationToken(token);
-
- System.out.printf("Token renewed successfully, expiry time: %s",
- expiryTime);
- }
-
- return null;
- }
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/token/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/token/package-info.java
deleted file mode 100644
index 5e03895..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/token/package-info.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * ozShell Class acts as the command line interface to the ozone Rest Client.
- */
-package org.apache.hadoop.ozone.web.ozShell.token;
-
-/**
- Ozone delegation token commands.
- **/
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/AddAclVolumeHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/AddAclVolumeHandler.java
deleted file mode 100644
index dcf3527..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/AddAclVolumeHandler.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.ozShell.volume;
-
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.security.acl.OzoneObj;
-import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import picocli.CommandLine;
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Parameters;
-
-import java.util.Objects;
-
-import static org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType.OZONE;
-
-/**
- * Add acl handler for volume.
- */
-@Command(name = "addacl",
- description = "Add a new ACL.")
-public class AddAclVolumeHandler extends Handler {
-
- @Parameters(arity = "1..1", description = Shell.OZONE_BUCKET_URI_DESCRIPTION)
- private String uri;
-
- @CommandLine.Option(names = {"--acl", "-a"},
- required = true,
- description = "The new ACL to be added.\n" +
- "Ex: user:user1:rw or group:hadoop:rw\n" +
- "r = READ, " +
- "w = WRITE, " +
- "c = CREATE, " +
- "d = DELETE, " +
- "l = LIST, " +
- "a = ALL, " +
- "n = NONE, " +
- "x = READ_ACL, " +
- "y = WRITE_ACL.")
- private String acl;
-
- @CommandLine.Option(names = {"--store", "-s"},
- required = false,
- description = "Store type. i.e OZONE or S3")
- private String storeType;
-
- /**
- * Executes the Client Calls.
- */
- @Override
- public Void call() throws Exception {
- Objects.requireNonNull(acl,
- "You need to specify a new ACL to be added.");
- OzoneAddress address = new OzoneAddress(uri);
- address.ensureVolumeAddress();
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- String volumeName = address.getVolumeName();
-
- if (isVerbose()) {
- System.out.printf("Volume Name : %s%n", volumeName);
- }
-
- OzoneObj obj = OzoneObjInfo.Builder.newBuilder()
- .setVolumeName(volumeName)
- .setResType(OzoneObj.ResourceType.VOLUME)
- .setStoreType(storeType == null ? OZONE :
- OzoneObj.StoreType.valueOf(storeType))
- .build();
-
- boolean result = client.getObjectStore().addAcl(obj,
- OzoneAcl.parseAcl(acl));
-
- String message = result
- ? ("ACL added successfully.")
- : ("ACL already exists.");
-
- System.out.println(message);
- }
-
- return null;
- }
-
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/CreateVolumeHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/CreateVolumeHandler.java
deleted file mode 100644
index f51b2db..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/CreateVolumeHandler.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.volume;
-
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.VolumeArgs;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.ObjectPrinter;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import org.apache.hadoop.security.UserGroupInformation;
-
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Option;
-import picocli.CommandLine.Parameters;
-
-/**
- * Executes the create volume call for the shell.
- */
-@Command(name = "create",
- description = "Creates a volume for the specified user")
-public class CreateVolumeHandler extends Handler {
-
- @Parameters(arity = "1..1", description = Shell.OZONE_VOLUME_URI_DESCRIPTION)
- private String uri;
-
- @Option(names = {"--user", "-u"},
- description = "Owner of of the volume")
- private String userName;
-
- @Option(names = {"--quota", "-q"},
- description =
- "Quota of the newly created volume (eg. 1G)")
- private String quota;
-
- @Option(names = {"--root"},
- description = "Development flag to execute the "
- + "command as the admin (hdfs) user.")
- private boolean root;
-
- /**
- * Executes the Create Volume.
- */
- @Override
- public Void call() throws Exception {
- if(userName == null) {
- userName = UserGroupInformation.getCurrentUser().getUserName();
- }
-
- OzoneAddress address = new OzoneAddress(uri);
- address.ensureVolumeAddress();
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- String volumeName = address.getVolumeName();
-
- if (isVerbose()) {
- System.out.printf("Volume name : %s%n", volumeName);
- }
-
- String rootName;
- if (root) {
- rootName = "hdfs";
- } else {
- rootName = UserGroupInformation.getCurrentUser().getShortUserName();
- }
-
- VolumeArgs.Builder volumeArgsBuilder = VolumeArgs.newBuilder()
- .setAdmin(rootName)
- .setOwner(userName);
- if (quota != null) {
- volumeArgsBuilder.setQuota(quota);
- }
- client.getObjectStore().createVolume(volumeName,
- volumeArgsBuilder.build());
-
- if (isVerbose()) {
- OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
- ObjectPrinter.printObjectAsJson(vol);
- }
- }
- return null;
- }
-
-}
-
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java
deleted file mode 100644
index 9d2b5ce..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.volume;
-
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Parameters;
-
-/**
- * Executes deleteVolume call for the shell.
- */
-@Command(name = "delete",
- description = "deletes a volume if it is empty")
-public class DeleteVolumeHandler extends Handler {
-
- @Parameters(arity = "1..1", description = Shell.OZONE_VOLUME_URI_DESCRIPTION)
- private String uri;
-
- /**
- * Executes the delete volume call.
- */
- @Override
- public Void call() throws Exception {
-
- OzoneAddress address = new OzoneAddress(uri);
- address.ensureVolumeAddress();
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- String volumeName = address.getVolumeName();
-
- if (isVerbose()) {
- System.out.printf("Volume name : %s%n", volumeName);
- }
-
- client.getObjectStore().deleteVolume(volumeName);
- System.out.printf("Volume %s is deleted%n", volumeName);
- }
-
- return null;
- }
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/GetAclVolumeHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/GetAclVolumeHandler.java
deleted file mode 100644
index 3e1ca57..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/GetAclVolumeHandler.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.ozShell.volume;
-
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.security.acl.OzoneObj;
-import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-import picocli.CommandLine;
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Parameters;
-
-import java.util.List;
-
-import static org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType.OZONE;
-
-/**
- * Get acl handler for volume.
- */
-@Command(name = "getacl",
- description = "List all ACLs.")
-public class GetAclVolumeHandler extends Handler {
-
- @Parameters(arity = "1..1", description = Shell.OZONE_BUCKET_URI_DESCRIPTION)
- private String uri;
-
- @CommandLine.Option(names = {"--store", "-s"},
- required = false,
- description = "Store type. i.e OZONE or S3")
- private String storeType;
-
- /**
- * Executes the Client Calls.
- */
- @Override
- public Void call() throws Exception {
- OzoneAddress address = new OzoneAddress(uri);
- address.ensureVolumeAddress();
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
- String volumeName = address.getVolumeName();
-
- if (isVerbose()) {
- System.out.printf("Volume Name : %s%n", volumeName);
- }
-
- OzoneObj obj = OzoneObjInfo.Builder.newBuilder()
- .setVolumeName(volumeName)
- .setResType(OzoneObj.ResourceType.VOLUME)
- .setStoreType(storeType == null ? OZONE :
- OzoneObj.StoreType.valueOf(storeType))
- .build();
- List<OzoneAcl> result = client.getObjectStore().getAcl(obj);
- System.out.printf("%s%n",
- JsonUtils.toJsonStringWithDefaultPrettyPrinter(result));
- }
-
- return null;
- }
-
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
deleted file mode 100644
index 10fb685..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.volume;
-
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.ObjectPrinter;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Parameters;
-
-/**
- * Executes volume Info calls.
- */
-@Command(name = "info",
- description = "returns information about a specific volume")
-public class InfoVolumeHandler extends Handler{
-
- @Parameters(arity = "1..1", description = Shell.OZONE_VOLUME_URI_DESCRIPTION)
- private String uri;
-
- /**
- * Executes volume Info.
- */
- @Override
- public Void call() throws Exception {
-
- OzoneAddress address = new OzoneAddress(uri);
- address.ensureVolumeAddress();
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- String volumeName = address.getVolumeName();
-
- OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
- ObjectPrinter.printObjectAsJson(vol);
- }
-
- return null;
- }
-
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/ListVolumeHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/ListVolumeHandler.java
deleted file mode 100644
index 6a09f12..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/ListVolumeHandler.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.volume;
-
-import java.util.Iterator;
-
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.ObjectPrinter;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import org.apache.hadoop.security.UserGroupInformation;
-
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Option;
-import picocli.CommandLine.Parameters;
-
-/**
- * Executes List Volume call.
- */
-@Command(name = "list",
- aliases = "ls",
- description = "List the volumes of a given user")
-public class ListVolumeHandler extends Handler {
-
- @Parameters(arity = "1..1",
- description = Shell.OZONE_VOLUME_URI_DESCRIPTION,
- defaultValue = "/")
- private String uri;
-
- @Option(names = {"--length", "-l"},
- description = "Limit of the max results",
- defaultValue = "100")
- private int maxVolumes;
-
- @Option(names = {"--start", "-s"},
- description = "The volume to start the listing from.\n" +
- "This will be excluded from the result.")
- private String startVolume;
-
- @Option(names = {"--prefix", "-p"},
- description = "Prefix to filter the volumes")
- private String prefix;
-
- @Option(names = {"--user", "-u"},
- description = "Owner of the volumes to list.")
- private String userName;
-
- /**
- * Executes the Client Calls.
- */
- @Override
- public Void call() throws Exception {
-
- OzoneAddress address = new OzoneAddress(uri);
- address.ensureRootAddress();
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- if (userName == null) {
- userName = UserGroupInformation.getCurrentUser().getUserName();
- }
-
- if (maxVolumes < 1) {
- throw new IllegalArgumentException(
- "the length should be a positive number");
- }
-
- Iterator<? extends OzoneVolume> volumeIterator;
- if (userName != null) {
- volumeIterator = client.getObjectStore()
- .listVolumesByUser(userName, prefix, startVolume);
- } else {
- volumeIterator = client.getObjectStore().listVolumes(prefix);
- }
-
- int counter = 0;
- while (maxVolumes > 0 && volumeIterator.hasNext()) {
- OzoneVolume next = volumeIterator.next();
- ObjectPrinter.printObjectAsJson(next);
- maxVolumes -= 1;
- counter++;
- }
-
- if (isVerbose()) {
- System.out.printf("Found : %d volumes for user : %s ", counter,
- userName);
- }
- }
-
- return null;
- }
-}
-
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/RemoveAclVolumeHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/RemoveAclVolumeHandler.java
deleted file mode 100644
index 741d5ff..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/RemoveAclVolumeHandler.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.ozShell.volume;
-
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.security.acl.OzoneObj;
-import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import picocli.CommandLine;
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Parameters;
-
-import java.util.Objects;
-
-import static org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType.OZONE;
-
-/**
- * Remove acl handler for volume.
- */
-@Command(name = "removeacl",
- description = "Remove an existing ACL.")
-public class RemoveAclVolumeHandler extends Handler {
-
- @Parameters(arity = "1..1", description = Shell.OZONE_BUCKET_URI_DESCRIPTION)
- private String uri;
-
- @CommandLine.Option(names = {"--acl", "-a"},
- required = true,
- description = "The ACL to be removed.\n" +
- "Ex: user:user1:rw or group:hadoop:rw\n" +
- "r = READ, " +
- "w = WRITE, " +
- "c = CREATE, " +
- "d = DELETE, " +
- "l = LIST, " +
- "a = ALL, " +
- "n = NONE, " +
- "x = READ_ACL, " +
- "y = WRITE_ACL.")
- private String acl;
-
- @CommandLine.Option(names = {"--store", "-s"},
- required = false,
- description = "Store type. i.e OZONE or S3")
- private String storeType;
-
- /**
- * Executes the Client Calls.
- */
- @Override
- public Void call() throws Exception {
- Objects.requireNonNull(acl,
- "You need to specify an ACL to be removed.");
- OzoneAddress address = new OzoneAddress(uri);
- address.ensureVolumeAddress();
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- String volumeName = address.getVolumeName();
-
- if (isVerbose()) {
- System.out.printf("Volume Name : %s%n", volumeName);
- }
-
- OzoneObj obj = OzoneObjInfo.Builder.newBuilder()
- .setVolumeName(volumeName)
- .setResType(OzoneObj.ResourceType.VOLUME)
- .setStoreType(storeType == null ? OZONE :
- OzoneObj.StoreType.valueOf(storeType))
- .build();
-
- boolean result = client.getObjectStore().removeAcl(obj,
- OzoneAcl.parseAcl(acl));
-
- String message = result
- ? ("ACL removed successfully.")
- : ("ACL doesn't exist.");
-
- System.out.println(message);
- }
-
- return null;
- }
-
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/SetAclVolumeHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/SetAclVolumeHandler.java
deleted file mode 100644
index a72198a..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/SetAclVolumeHandler.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.ozShell.volume;
-
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.security.acl.OzoneObj;
-import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import picocli.CommandLine;
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Parameters;
-
-import java.util.Objects;
-
-import static org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType.OZONE;
-
-/**
- * Set acl handler for volume.
- */
-@Command(name = "setacl",
- description = "Set one or more ACLs, replacing the existing ones.")
-public class SetAclVolumeHandler extends Handler {
-
- @Parameters(arity = "1..1", description = Shell.OZONE_BUCKET_URI_DESCRIPTION)
- private String uri;
-
- @CommandLine.Option(names = {"--acls", "-al"},
- required = true,
- description = "A comma separated list of ACLs to be set.\n" +
- "Ex: user:user1:rw,user:user2:a,group:hadoop:a\n" +
- "r = READ, " +
- "w = WRITE, " +
- "c = CREATE, " +
- "d = DELETE, " +
- "l = LIST, " +
- "a = ALL, " +
- "n = NONE, " +
- "x = READ_ACL, " +
- "y = WRITE_ACL.")
- private String acls;
-
- @CommandLine.Option(names = {"--store", "-s"},
- required = false,
- description = "Store type. i.e OZONE or S3")
- private String storeType;
-
- /**
- * Executes the Client Calls.
- */
- @Override
- public Void call() throws Exception {
- Objects.requireNonNull(acls,
- "You need to specify one or more ACLs to be set.");
- OzoneAddress address = new OzoneAddress(uri);
- address.ensureVolumeAddress();
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- String volumeName = address.getVolumeName();
- String bucketName = address.getBucketName();
-
- if (isVerbose()) {
- System.out.printf("Volume Name : %s%n", volumeName);
- System.out.printf("Bucket Name : %s%n", bucketName);
- }
-
- OzoneObj obj = OzoneObjInfo.Builder.newBuilder()
- .setBucketName(bucketName)
- .setVolumeName(volumeName)
- .setResType(OzoneObj.ResourceType.VOLUME)
- .setStoreType(storeType == null ? OZONE :
- OzoneObj.StoreType.valueOf(storeType))
- .build();
-
- boolean result = client.getObjectStore().setAcl(obj,
- OzoneAcl.parseAcls(acls));
-
- String message = result
- ? ("ACL(s) set successfully.")
- : ("ACL(s) already set.");
-
- System.out.println(message);
- }
-
- return null;
- }
-
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java
deleted file mode 100644
index 7e23ce0..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.volume;
-
-import org.apache.hadoop.hdds.client.OzoneQuota;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.ObjectPrinter;
-import org.apache.hadoop.ozone.web.ozShell.OzoneAddress;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Option;
-import picocli.CommandLine.Parameters;
-
-/**
- * Executes update volume calls.
- */
-@Command(name = "update",
- description = "Updates parameter of the volumes")
-public class UpdateVolumeHandler extends Handler {
-
- @Parameters(arity = "1..1", description = Shell.OZONE_VOLUME_URI_DESCRIPTION)
- private String uri;
-
- @Option(names = {"--user"},
- description = "Owner of the volume to set")
- private String ownerName;
-
- @Option(names = {"--quota"},
- description = "Quota of the volume to set"
- + "(eg. 1G)")
- private String quota;
-
- /**
- * Executes the Client Calls.
- */
- @Override
- public Void call() throws Exception {
-
- OzoneAddress address = new OzoneAddress(uri);
- address.ensureVolumeAddress();
- try (OzoneClient client =
- address.createClient(createOzoneConfiguration())) {
-
- String volumeName = address.getVolumeName();
-
- OzoneVolume volume = client.getObjectStore().getVolume(volumeName);
- if (quota != null && !quota.isEmpty()) {
- volume.setQuota(OzoneQuota.parseQuota(quota));
- }
-
- if (ownerName != null && !ownerName.isEmpty()) {
- volume.setOwner(ownerName);
- }
-
- ObjectPrinter.printObjectAsJson(volume);
- }
- return null;
- }
-}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/package-info.java
deleted file mode 100644
index fc19274..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * Netty-based HTTP server implementation for Ozone.
- */
-package org.apache.hadoop.ozone.web.ozShell.volume;
\ No newline at end of file
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/package-info.java
deleted file mode 100644
index 1a7275c..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web;
-
-/**
- * This package contains generic class for the internal http server
- * and REST interfaces.
- */
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java
index 320c3a5..4e62eb8 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java
@@ -40,6 +40,7 @@
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
@@ -353,8 +354,16 @@
.setNodes(Arrays.asList(dnFour, dnFive, dnSix))
.build();
- Mockito.when(containerClient.getContainerWithPipeline(1L))
- .thenReturn(new ContainerWithPipeline(null, pipelineTwo));
+ List<Long> containerIDs = new ArrayList<>();
+ containerIDs.add(1L);
+
+ List<ContainerWithPipeline> cps = new ArrayList<>();
+ ContainerInfo ci = Mockito.mock(ContainerInfo.class);
+ Mockito.when(ci.getContainerID()).thenReturn(1L);
+ cps.add(new ContainerWithPipeline(ci, pipelineTwo));
+
+ Mockito.when(containerClient.getContainerWithPipelineBatch(containerIDs))
+ .thenReturn(cps);
final OmVolumeArgs volumeArgs = OmVolumeArgs.newBuilder()
.setVolume("volumeOne")
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMStorage.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMStorage.java
index d752ec1..48873cb 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMStorage.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMStorage.java
@@ -18,13 +18,14 @@
import java.io.File;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.commons.io.FileUtils;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -45,7 +46,7 @@
final File testDir = createTestDir();
final File dbDir = new File(testDir, "omDbDir");
final File metaDir = new File(testDir, "metaDir"); // should be ignored.
- final Configuration conf = new OzoneConfiguration();
+ final ConfigurationSource conf = new OzoneConfiguration();
conf.set(OMConfigKeys.OZONE_OM_DB_DIRS, dbDir.getPath());
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.getPath());
@@ -65,7 +66,7 @@
public void testGetOmDbDirWithFallback() {
final File testDir = createTestDir();
final File metaDir = new File(testDir, "metaDir");
- final Configuration conf = new OzoneConfiguration();
+ final ConfigurationSource conf = new OzoneConfiguration();
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.getPath());
try {
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
index dc2187c..8975a24 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
@@ -149,8 +149,12 @@
TreeSet<String> volumeABucketsPrefixWithOzoneOwner = new TreeSet<>();
TreeSet<String> volumeABucketsPrefixWithHadoopOwner = new TreeSet<>();
- for (int i=1; i<= 100; i++) {
- if (i % 2 == 0) {
+
+ // Add exact name in prefixBucketNameWithOzoneOwner without postfix.
+ volumeABucketsPrefixWithOzoneOwner.add(prefixBucketNameWithOzoneOwner);
+ addBucketsToCache(volumeName1, prefixBucketNameWithOzoneOwner);
+ for (int i = 1; i < 100; i++) {
+ if (i % 2 == 0) { // This part adds 49 buckets.
volumeABucketsPrefixWithOzoneOwner.add(
prefixBucketNameWithOzoneOwner + i);
addBucketsToCache(volumeName1, prefixBucketNameWithOzoneOwner + i);
@@ -165,8 +169,12 @@
TreeSet<String> volumeBBucketsPrefixWithOzoneOwner = new TreeSet<>();
TreeSet<String> volumeBBucketsPrefixWithHadoopOwner = new TreeSet<>();
TestOMRequestUtils.addVolumeToDB(volumeName2, omMetadataManager);
- for (int i=1; i<= 100; i++) {
- if (i % 2 == 0) {
+
+ // Add exact name in prefixBucketNameWithOzoneOwner without postfix.
+ volumeBBucketsPrefixWithOzoneOwner.add(prefixBucketNameWithOzoneOwner);
+ addBucketsToCache(volumeName2, prefixBucketNameWithOzoneOwner);
+ for (int i = 1; i < 100; i++) {
+ if (i % 2 == 0) { // This part adds 49 buckets.
volumeBBucketsPrefixWithOzoneOwner.add(
prefixBucketNameWithOzoneOwner + i);
addBucketsToCache(volumeName2, prefixBucketNameWithOzoneOwner + i);
@@ -182,7 +190,10 @@
omMetadataManager.listBuckets(volumeName1,
null, prefixBucketNameWithOzoneOwner, 100);
- Assert.assertEquals(omBucketInfoList.size(), 50);
+ // Cause adding a exact name in prefixBucketNameWithOzoneOwner
+ // and another 49 buckets, so if we list buckets with --prefix
+ // prefixBucketNameWithOzoneOwner, we should get 50 buckets.
+ Assert.assertEquals(omBucketInfoList.size(), 50);
for (OmBucketInfo omBucketInfo : omBucketInfoList) {
Assert.assertTrue(omBucketInfo.getBucketName().startsWith(
@@ -220,7 +231,10 @@
omBucketInfoList = omMetadataManager.listBuckets(volumeName2,
null, prefixBucketNameWithHadoopOwner, 100);
- Assert.assertEquals(omBucketInfoList.size(), 50);
+ // Cause adding a exact name in prefixBucketNameWithOzoneOwner
+ // and another 49 buckets, so if we list buckets with --prefix
+ // prefixBucketNameWithOzoneOwner, we should get 50 buckets.
+ Assert.assertEquals(omBucketInfoList.size(), 50);
for (OmBucketInfo omBucketInfo : omBucketInfoList) {
Assert.assertTrue(omBucketInfo.getBucketName().startsWith(
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHttpServer.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHttpServer.java
index f35d3d3..2d0a72b 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHttpServer.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHttpServer.java
@@ -18,8 +18,15 @@
package org.apache.hadoop.ozone.om;
-import org.apache.hadoop.conf.Configuration;
+import java.io.File;
+import java.net.InetSocketAddress;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.Arrays;
+import java.util.Collection;
+
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdfs.web.URLConnectionFactory;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.http.HttpConfig.Policy;
@@ -27,6 +34,7 @@
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
import org.apache.hadoop.test.GenericTestUtils;
+
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -35,13 +43,6 @@
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
-import java.io.File;
-import java.net.InetSocketAddress;
-import java.net.URL;
-import java.net.URLConnection;
-import java.util.Arrays;
-import java.util.Collection;
-
/**
* Test http server of OM with various HTTP option.
*/
@@ -51,7 +52,7 @@
.getTempPath(TestOzoneManagerHttpServer.class.getSimpleName());
private static String keystoresDir;
private static String sslConfDir;
- private static Configuration conf;
+ private static OzoneConfiguration conf;
private static URLConnectionFactory connectionFactory;
@Parameters public static Collection<Object[]> policy() {
@@ -73,7 +74,7 @@
File base = new File(BASEDIR);
FileUtil.fullyDelete(base);
base.mkdirs();
- conf = new Configuration();
+ conf = new OzoneConfiguration();
keystoresDir = new File(BASEDIR).getAbsolutePath();
sslConfDir = KeyStoreTestUtil.getClasspathDir(
TestOzoneManagerHttpServer.class);
@@ -87,6 +88,7 @@
}
@AfterClass public static void tearDown() throws Exception {
+ connectionFactory.destroy();
FileUtil.fullyDelete(new File(BASEDIR));
KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir);
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java
deleted file mode 100644
index 0513876..0000000
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.hadoop.ozone.om;
-
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.server.ServerUtils;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.File;
-import java.io.IOException;
-
-import static org.junit.Assert.*;
-
-/**
- * Tests for S3 Bucket Manager.
- */
-public class TestS3BucketManager {
- @Rule
- public ExpectedException thrown = ExpectedException.none();
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
- private OzoneConfiguration conf;
- private OmMetadataManagerImpl metaMgr;
- private BucketManager bucketManager;
- private VolumeManager volumeManager;
-
- @Before
- public void init() throws IOException {
- conf = new OzoneConfiguration();
- File newFolder = folder.newFolder();
- if (!newFolder.exists()) {
- Assert.assertTrue(newFolder.mkdirs());
- }
- ServerUtils.setOzoneMetaDirPath(conf, newFolder.toString());
- metaMgr = new OmMetadataManagerImpl(conf);
- volumeManager = new VolumeManagerImpl(metaMgr, conf);
- bucketManager = new BucketManagerImpl(metaMgr);
- }
-
- @Test
- public void testOzoneVolumeNameForUser() throws IOException {
- S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr,
- volumeManager, bucketManager);
- String userName = OzoneConsts.OZONE;
- String volumeName = s3BucketManager.getOzoneVolumeNameForUser(userName);
- assertEquals(OzoneConsts.OM_S3_VOLUME_PREFIX + userName, volumeName);
- }
-
- @Test
- public void testOzoneVolumeNameForUserFails() throws IOException {
- S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr,
- volumeManager, bucketManager);
- String userName = null;
- try {
- String volumeName = s3BucketManager.getOzoneVolumeNameForUser(userName);
- fail("testOzoneVolumeNameForUserFails failed");
- } catch (NullPointerException ex) {
- GenericTestUtils.assertExceptionContains("UserName cannot be null", ex);
- }
-
- }
-
- @Test
- public void testGetS3BucketMapping() throws IOException {
- S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr,
- volumeManager, bucketManager);
- String userName = "bilbo";
- metaMgr.getS3Table().put("newBucket",
- s3BucketManager.formatOzoneVolumeName(userName) + "/newBucket");
- String mapping = s3BucketManager.getOzoneBucketMapping("newBucket");
- Assert.assertTrue(mapping.startsWith("s3bilbo/"));
- Assert.assertTrue(mapping.endsWith("/newBucket"));
- }
-
- @Test
- public void testGetOzoneNames() throws IOException {
- S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr,
- volumeManager, bucketManager);
- String userName = "batman";
- String s3BucketName = "gotham";
- metaMgr.getS3Table().put(s3BucketName,
- s3BucketManager.formatOzoneVolumeName(userName) + "/" + s3BucketName);
- String volumeName = s3BucketManager.getOzoneVolumeName(s3BucketName);
- Assert.assertTrue(volumeName.equalsIgnoreCase("s3"+userName));
- String bucketName =s3BucketManager.getOzoneBucketName(s3BucketName);
- Assert.assertTrue(bucketName.equalsIgnoreCase(s3BucketName));
- // try to get a bucket that does not exist.
- thrown.expectMessage("No such S3 bucket.");
- s3BucketManager.getOzoneBucketMapping("raven");
-
- }
-}
\ No newline at end of file
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashService.java
index 087bf17..cf9e626 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashService.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashService.java
@@ -55,6 +55,7 @@
public TemporaryFolder tempFolder = new TemporaryFolder();
private KeyManager keyManager;
+ private OmMetadataManagerImpl omMetadataManager;
private String volumeName;
private String bucketName;
@@ -69,8 +70,8 @@
System.setProperty(DBConfigFromFile.CONFIG_DIR, "/");
ServerUtils.setOzoneMetaDirPath(configuration, folder.toString());
- OmMetadataManagerImpl omMetadataManager =
- new OmMetadataManagerImpl(configuration);
+ omMetadataManager = new OmMetadataManagerImpl(configuration);
+
keyManager = new KeyManagerImpl(
new ScmBlockLocationTestingClient(null, null, 0),
omMetadataManager, configuration, UUID.randomUUID().toString(), null);
@@ -86,11 +87,9 @@
String destinationBucket = "destBucket";
createAndDeleteKey(keyName);
- /* TODO:HDDS-2424. */
- // boolean recoverOperation =
- // ozoneManager.recoverTrash(
- // volumeName, bucketName, keyName, destinationBucket);
- // Assert.assertTrue(recoverOperation);
+ boolean recoverOperation = omMetadataManager
+ .recoverTrash(volumeName, bucketName, keyName, destinationBucket);
+ Assert.assertTrue(recoverOperation);
}
private void createAndDeleteKey(String keyName) throws IOException {
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/codec/TestOmMultipartKeyInfoCodec.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/codec/TestOmMultipartKeyInfoCodec.java
similarity index 100%
rename from hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/codec/TestOmMultipartKeyInfoCodec.java
rename to hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/codec/TestOmMultipartKeyInfoCodec.java
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/codec/TestOmPrefixInfoCodec.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/codec/TestOmPrefixInfoCodec.java
similarity index 100%
rename from hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/codec/TestOmPrefixInfoCodec.java
rename to hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/codec/TestOmPrefixInfoCodec.java
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/codec/TestS3SecretValueCodec.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/codec/TestS3SecretValueCodec.java
similarity index 100%
rename from hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/codec/TestS3SecretValueCodec.java
rename to hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/codec/TestS3SecretValueCodec.java
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/codec/package-info.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/codec/package-info.java
similarity index 100%
rename from hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/codec/package-info.java
rename to hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/codec/package-info.java
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java
index 338f04c..12de31a 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java
@@ -71,8 +71,11 @@
OzoneManagerRatisSnapshot ozoneManagerRatisSnapshot = index -> {
lastAppliedIndex = index.get(index.size() - 1);
};
- doubleBuffer = new OzoneManagerDoubleBuffer(omMetadataManager,
- ozoneManagerRatisSnapshot);
+ doubleBuffer = new OzoneManagerDoubleBuffer.Builder()
+ .setOmMetadataManager(omMetadataManager)
+ .setOzoneManagerRatisSnapShot(ozoneManagerRatisSnapshot)
+ .enableRatis(true)
+ .build();
}
@After
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithOMResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithOMResponse.java
index d0f4b08..5c274f0 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithOMResponse.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithOMResponse.java
@@ -103,8 +103,11 @@
ozoneManagerRatisSnapshot = index -> {
lastAppliedIndex = index.get(index.size() - 1);
};
- doubleBuffer = new OzoneManagerDoubleBuffer(omMetadataManager,
- ozoneManagerRatisSnapshot);
+ doubleBuffer = new OzoneManagerDoubleBuffer.Builder().
+ setOmMetadataManager(omMetadataManager).
+ setOzoneManagerRatisSnapShot(ozoneManagerRatisSnapshot)
+ .enableRatis(true)
+ .build();
ozoneManagerDoubleBufferHelper = doubleBuffer::add;
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerStateMachine.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerStateMachine.java
index 33ac538..b3ff9f8 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerStateMachine.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerStateMachine.java
@@ -43,7 +43,7 @@
when(ozoneManager.getSnapshotInfo()).thenReturn(
Mockito.mock(OMRatisSnapshotInfo.class));
ozoneManagerStateMachine =
- new OzoneManagerStateMachine(ozoneManagerRatisServer);
+ new OzoneManagerStateMachine(ozoneManagerRatisServer, false);
}
@Test
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java
index 82fa795..c5aa9fe 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java
@@ -35,7 +35,6 @@
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
-import org.apache.hadoop.ozone.om.request.s3.bucket.S3BucketCreateRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.MultipartUploadAbortRequest;
@@ -241,15 +240,6 @@
addVolumeToDB(volumeName, UUID.randomUUID().toString(), omMetadataManager);
}
- public static void addS3BucketToDB(String volumeName, String s3BucketName,
- OMMetadataManager omMetadataManager) throws Exception {
- omMetadataManager.getS3Table().put(s3BucketName,
- S3BucketCreateRequest.formatS3MappingName(volumeName, s3BucketName));
- OmBucketInfo omBucketInfo = OmBucketInfo.newBuilder()
- .setVolumeName(volumeName).setBucketName(s3BucketName).build();
- addBucketToOM(omMetadataManager, omBucketInfo);
- }
-
/**
* Add volume creation entry to OM DB.
* @param volumeName
@@ -311,30 +301,6 @@
.setClientId(UUID.randomUUID().toString()).build();
}
- public static OzoneManagerProtocolProtos.OMRequest createS3BucketRequest(
- String userName, String s3BucketName) {
- OzoneManagerProtocolProtos.S3CreateBucketRequest request =
- OzoneManagerProtocolProtos.S3CreateBucketRequest.newBuilder()
- .setUserName(userName)
- .setS3Bucketname(s3BucketName).build();
-
- return OzoneManagerProtocolProtos.OMRequest.newBuilder()
- .setCreateS3BucketRequest(request)
- .setCmdType(OzoneManagerProtocolProtos.Type.CreateS3Bucket)
- .setClientId(UUID.randomUUID().toString()).build();
- }
-
- public static OzoneManagerProtocolProtos.OMRequest deleteS3BucketRequest(
- String s3BucketName) {
- OzoneManagerProtocolProtos.S3DeleteBucketRequest request =
- OzoneManagerProtocolProtos.S3DeleteBucketRequest.newBuilder()
- .setS3BucketName(s3BucketName).build();
- return OzoneManagerProtocolProtos.OMRequest.newBuilder()
- .setDeleteS3BucketRequest(request)
- .setCmdType(OzoneManagerProtocolProtos.Type.DeleteS3Bucket)
- .setClientId(UUID.randomUUID().toString()).build();
- }
-
public static List< HddsProtos.KeyValue> getMetadataList() {
List<HddsProtos.KeyValue> metadataList = new ArrayList<>();
metadataList.add(HddsProtos.KeyValue.newBuilder().setKey("key1").setValue(
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java
index 561bd14..ab5ad6b 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java
@@ -21,6 +21,8 @@
import java.util.UUID;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.test.LambdaTestUtils;
import org.junit.Assert;
import org.junit.Test;
@@ -48,6 +50,9 @@
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
doPreExecute(volumeName, bucketName);
+ // Verify invalid bucket name throws exception
+ LambdaTestUtils.intercept(OMException.class, "Invalid bucket name: b1",
+ () -> doPreExecute("volume1", "b1"));
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequest.java
index 604fdce..b714375 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequest.java
@@ -20,7 +20,7 @@
import java.util.UUID;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/bucket/TestS3BucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/bucket/TestS3BucketCreateRequest.java
deleted file mode 100644
index 9ad27ea..0000000
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/bucket/TestS3BucketCreateRequest.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.ozone.om.request.s3.bucket;
-
-import java.util.UUID;
-
-import org.apache.commons.lang.RandomStringUtils;
-import org.junit.Assert;
-import org.junit.Test;
-
-import org.apache.hadoop.ozone.om.exceptions.OMException;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
- .OMRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
- .OMResponse;
-import org.apache.hadoop.ozone.om.request.TestOMRequestUtils;
-import org.apache.hadoop.ozone.om.response.OMClientResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
-
-import static org.junit.Assert.fail;
-
-/**
- * Tests S3BucketCreateRequest class, which handles S3 CreateBucket request.
- */
-public class TestS3BucketCreateRequest extends TestS3BucketRequest {
-
- @Test
- public void testPreExecute() throws Exception {
- String userName = UUID.randomUUID().toString();
- String s3BucketName = UUID.randomUUID().toString();
- doPreExecute(userName, s3BucketName);
- }
-
- @Test
- public void testPreExecuteInvalidBucketLength() throws Exception {
- String userName = UUID.randomUUID().toString();
-
- // set bucket name which is less than 3 characters length
- String s3BucketName = RandomStringUtils.randomAlphabetic(2);
-
- try {
- doPreExecute(userName, s3BucketName);
- fail("testPreExecuteInvalidBucketLength failed");
- } catch (OMException ex) {
- GenericTestUtils.assertExceptionContains("S3_BUCKET_INVALID_LENGTH", ex);
- }
-
- // set bucket name which is greater than 63 characters length
- s3BucketName = RandomStringUtils.randomAlphabetic(64);
-
- try {
- doPreExecute(userName, s3BucketName);
- fail("testPreExecuteInvalidBucketLength failed");
- } catch (OMException ex) {
- GenericTestUtils.assertExceptionContains("S3_BUCKET_INVALID_LENGTH", ex);
- }
- }
-
- @Test
- public void testValidateAndUpdateCache() throws Exception {
- String userName = UUID.randomUUID().toString();
- String s3BucketName = UUID.randomUUID().toString();
-
- S3BucketCreateRequest s3BucketCreateRequest = doPreExecute(userName,
- s3BucketName);
-
- doValidateAndUpdateCache(userName, s3BucketName,
- s3BucketCreateRequest.getOmRequest());
- }
-
- @Test
- public void testValidateAndUpdateCacheWithS3BucketAlreadyExists()
- throws Exception {
- String userName = UUID.randomUUID().toString();
- String s3BucketName = UUID.randomUUID().toString();
-
- TestOMRequestUtils.addS3BucketToDB(
- S3BucketCreateRequest.formatOzoneVolumeName(userName), s3BucketName,
- omMetadataManager);
-
- S3BucketCreateRequest s3BucketCreateRequest =
- doPreExecute(userName, s3BucketName);
-
-
- // Try create same bucket again
- OMClientResponse omClientResponse =
- s3BucketCreateRequest.validateAndUpdateCache(ozoneManager, 2,
- ozoneManagerDoubleBufferHelper);
-
- OMResponse omResponse = omClientResponse.getOMResponse();
- Assert.assertNotNull(omResponse.getCreateBucketResponse());
- Assert.assertEquals(
- OzoneManagerProtocolProtos.Status.S3_BUCKET_ALREADY_EXISTS,
- omResponse.getStatus());
- }
-
- @Test
- public void testValidateAndUpdateCacheWithBucketAlreadyExists()
- throws Exception {
- String userName = UUID.randomUUID().toString();
- String s3BucketName = UUID.randomUUID().toString();
-
- S3BucketCreateRequest s3BucketCreateRequest =
- doPreExecute(userName, s3BucketName);
-
- TestOMRequestUtils.addVolumeAndBucketToDB(
- s3BucketCreateRequest.formatOzoneVolumeName(userName),
- s3BucketName, omMetadataManager);
-
- // Try create same bucket again
- OMClientResponse omClientResponse =
- s3BucketCreateRequest.validateAndUpdateCache(ozoneManager, 2,
- ozoneManagerDoubleBufferHelper);
-
- OMResponse omResponse = omClientResponse.getOMResponse();
- Assert.assertNotNull(omResponse.getCreateBucketResponse());
- Assert.assertEquals(OzoneManagerProtocolProtos.Status.BUCKET_ALREADY_EXISTS,
- omResponse.getStatus());
- }
-
- private S3BucketCreateRequest doPreExecute(String userName,
- String s3BucketName) throws Exception {
- OMRequest originalRequest =
- TestOMRequestUtils.createS3BucketRequest(userName, s3BucketName);
-
- S3BucketCreateRequest s3BucketCreateRequest =
- new S3BucketCreateRequest(originalRequest);
-
- OMRequest modifiedRequest = s3BucketCreateRequest.preExecute(ozoneManager);
- // Modification time will be set, so requests should not be equal.
- Assert.assertNotEquals(originalRequest, modifiedRequest);
- return new S3BucketCreateRequest(modifiedRequest);
- }
-
- private void doValidateAndUpdateCache(String userName, String s3BucketName,
- OMRequest modifiedRequest) throws Exception {
-
- // As we have not still called validateAndUpdateCache, get() should
- // return null.
-
- Assert.assertNull(omMetadataManager.getS3Table().get(s3BucketName));
- S3BucketCreateRequest s3BucketCreateRequest =
- new S3BucketCreateRequest(modifiedRequest);
-
-
- OMClientResponse omClientResponse =
- s3BucketCreateRequest.validateAndUpdateCache(ozoneManager, 1,
- ozoneManagerDoubleBufferHelper);
-
- // As now after validateAndUpdateCache it should add entry to cache, get
- // should return non null value.
-
- Assert.assertNotNull(omMetadataManager.getS3Table().get(s3BucketName));
-
- String bucketKey =
- omMetadataManager.getBucketKey(
- s3BucketCreateRequest.formatOzoneVolumeName(userName),
- s3BucketName);
-
- // check ozone bucket entry is created or not.
- Assert.assertNotNull(omMetadataManager.getBucketTable().get(bucketKey));
-
- String volumeKey = omMetadataManager.getVolumeKey(
- s3BucketCreateRequest.formatOzoneVolumeName(userName));
-
- // Check volume entry is created or not.
- Assert.assertNotNull(omMetadataManager.getVolumeTable().get(volumeKey));
-
- // check om response.
- Assert.assertEquals(OzoneManagerProtocolProtos.Status.OK,
- omClientResponse.getOMResponse().getStatus());
- Assert.assertEquals(OzoneManagerProtocolProtos.Type.CreateS3Bucket,
- omClientResponse.getOMResponse().getCmdType());
- }
-
- @Test
- public void testReplayRequest() throws Exception {
-
- String userName = UUID.randomUUID().toString();
- String s3BucketName = UUID.randomUUID().toString();
-
- // Execute the original request
- S3BucketCreateRequest s3BucketCreateRequest =
- doPreExecute(userName, s3BucketName);
- s3BucketCreateRequest.validateAndUpdateCache(ozoneManager, 2,
- ozoneManagerDoubleBufferHelper);
-
- // Replay the transaction - Execute the same request again
- OMClientResponse omClientResponse =
- s3BucketCreateRequest.validateAndUpdateCache(ozoneManager, 2,
- ozoneManagerDoubleBufferHelper);
-
- // Replay should result in Replay response
- Assert.assertEquals(OzoneManagerProtocolProtos.Status.REPLAY,
- omClientResponse.getOMResponse().getStatus());
- }
-}
-
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/bucket/TestS3BucketDeleteRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/bucket/TestS3BucketDeleteRequest.java
deleted file mode 100644
index 7fe87ad..0000000
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/bucket/TestS3BucketDeleteRequest.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.ozone.om.request.s3.bucket;
-
-import java.util.UUID;
-
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.junit.Assert;
-import org.junit.Test;
-
-import org.apache.hadoop.ozone.om.exceptions.OMException;
-import org.apache.hadoop.ozone.om.request.TestOMRequestUtils;
-import org.apache.hadoop.ozone.om.response.OMClientResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
- .OMRequest;
-import org.apache.hadoop.test.GenericTestUtils;
-
-import static org.junit.Assert.fail;
-
-/**
- * Tests S3BucketDelete Request.
- */
-public class TestS3BucketDeleteRequest extends TestS3BucketRequest {
-
- @Test
- public void testPreExecute() throws Exception {
- String s3BucketName = UUID.randomUUID().toString();
- doPreExecute(s3BucketName);
- }
-
- @Test
- public void testValidateAndUpdateCache() throws Exception {
- String s3BucketName = UUID.randomUUID().toString();
- OMRequest omRequest = doPreExecute(s3BucketName);
-
- // Add s3Bucket to s3Bucket table.
- TestOMRequestUtils.addS3BucketToDB(OzoneConsts.OZONE, s3BucketName,
- omMetadataManager);
-
- S3BucketDeleteRequest s3BucketDeleteRequest =
- new S3BucketDeleteRequest(omRequest);
-
- OMClientResponse s3BucketDeleteResponse =
- s3BucketDeleteRequest.validateAndUpdateCache(ozoneManager, 1L,
- ozoneManagerDoubleBufferHelper);
-
- Assert.assertEquals(OzoneManagerProtocolProtos.Status.OK,
- s3BucketDeleteResponse.getOMResponse().getStatus());
- }
-
- @Test
- public void testValidateAndUpdateCacheWithS3BucketNotFound()
- throws Exception {
- String s3BucketName = UUID.randomUUID().toString();
- OMRequest omRequest = doPreExecute(s3BucketName);
-
- S3BucketDeleteRequest s3BucketDeleteRequest =
- new S3BucketDeleteRequest(omRequest);
-
- OMClientResponse s3BucketDeleteResponse =
- s3BucketDeleteRequest.validateAndUpdateCache(ozoneManager, 1L,
- ozoneManagerDoubleBufferHelper);
-
- Assert.assertEquals(OzoneManagerProtocolProtos.Status.S3_BUCKET_NOT_FOUND,
- s3BucketDeleteResponse.getOMResponse().getStatus());
- }
-
- @Test
- public void testPreExecuteInvalidBucketLength() throws Exception {
- // set bucket name which is less than 3 characters length
- String s3BucketName = RandomStringUtils.randomAlphabetic(2);
-
- try {
- doPreExecute(s3BucketName);
- fail("testPreExecuteInvalidBucketLength failed");
- } catch (OMException ex) {
- GenericTestUtils.assertExceptionContains("S3_BUCKET_INVALID_LENGTH", ex);
- }
-
- // set bucket name which is less than 3 characters length
- s3BucketName = RandomStringUtils.randomAlphabetic(65);
-
- try {
- doPreExecute(s3BucketName);
- fail("testPreExecuteInvalidBucketLength failed");
- } catch (OMException ex) {
- GenericTestUtils.assertExceptionContains("S3_BUCKET_INVALID_LENGTH", ex);
- }
- }
-
- private OMRequest doPreExecute(String s3BucketName) throws Exception {
- OMRequest omRequest =
- TestOMRequestUtils.deleteS3BucketRequest(s3BucketName);
-
- S3BucketDeleteRequest s3BucketDeleteRequest =
- new S3BucketDeleteRequest(omRequest);
-
- OMRequest modifiedOMRequest =
- s3BucketDeleteRequest.preExecute(ozoneManager);
-
- // As user name will be set both should not be equal.
- Assert.assertNotEquals(omRequest, modifiedOMRequest);
-
- return modifiedOMRequest;
- }
-
- @Test
- public void testReplayRequest() throws Exception {
-
- String userName = UUID.randomUUID().toString();
- String bucketName = UUID.randomUUID().toString();
-
- // Execute CreateS3Bucket request
- S3BucketCreateRequest s3BucketCreateRequest = new S3BucketCreateRequest(
- TestOMRequestUtils.createS3BucketRequest(userName, bucketName));
- s3BucketCreateRequest.preExecute(ozoneManager);
- s3BucketCreateRequest.validateAndUpdateCache(ozoneManager, 2,
- ozoneManagerDoubleBufferHelper);
-
- // Execute the original DeleteBucket request
- OMRequest omRequest = TestOMRequestUtils.deleteS3BucketRequest(bucketName);
- S3BucketDeleteRequest s3BucketDeleteRequest = new S3BucketDeleteRequest(
- omRequest);
- s3BucketDeleteRequest.preExecute(ozoneManager);
- s3BucketDeleteRequest.validateAndUpdateCache(ozoneManager, 4,
- ozoneManagerDoubleBufferHelper);
-
- // Create the bucket again
- s3BucketCreateRequest.preExecute(ozoneManager);
- s3BucketCreateRequest.validateAndUpdateCache(ozoneManager, 10,
- ozoneManagerDoubleBufferHelper);
-
- // Replay the delete transaction - Execute the same request again
- OMClientResponse omClientResponse =
- s3BucketDeleteRequest.validateAndUpdateCache(ozoneManager, 4,
- ozoneManagerDoubleBufferHelper);
-
- // Replay should result in Replay response
- Assert.assertEquals(OzoneManagerProtocolProtos.Status.REPLAY,
- omClientResponse.getOMResponse().getStatus());
- }
-}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/bucket/TestS3BucketRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/bucket/TestS3BucketRequest.java
deleted file mode 100644
index 9253ed0..0000000
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/bucket/TestS3BucketRequest.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.ozone.om.request.s3.bucket;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.rules.TemporaryFolder;
-import org.mockito.Mockito;
-
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.audit.AuditLogger;
-import org.apache.hadoop.ozone.audit.AuditMessage;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
-import org.apache.hadoop.ozone.om.OMMetadataManager;
-import org.apache.hadoop.ozone.om.OMMetrics;
-import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
-import org.apache.hadoop.ozone.om.OzoneManager;
-import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
-
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.when;
-
-/**
- * Base test class for S3 Bucket request.
- */
-@SuppressWarnings("visibilityModifier")
-public class TestS3BucketRequest {
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
-
- protected OzoneManager ozoneManager;
- protected OMMetrics omMetrics;
- protected OMMetadataManager omMetadataManager;
- protected AuditLogger auditLogger;
-
- // Just setting ozoneManagerDoubleBuffer which does nothing.
- protected OzoneManagerDoubleBufferHelper ozoneManagerDoubleBufferHelper =
- ((response, transactionIndex) -> {
- return null;
- });
-
- @Before
- public void setup() throws Exception {
-
- ozoneManager = Mockito.mock(OzoneManager.class);
- omMetrics = OMMetrics.create();
- OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
- ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS,
- folder.newFolder().getAbsolutePath());
- omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration);
- when(ozoneManager.getMetrics()).thenReturn(omMetrics);
- when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager);
- when(ozoneManager.isRatisEnabled()).thenReturn(true);
- auditLogger = Mockito.mock(AuditLogger.class);
- when(ozoneManager.getAuditLogger()).thenReturn(auditLogger);
- Mockito.doNothing().when(auditLogger).logWrite(any(AuditMessage.class));
- }
-
- @After
- public void stop() {
- omMetrics.unRegister();
- Mockito.framework().clearInlineMocks();
- }
-}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/bucket/package-info.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/bucket/package-info.java
deleted file mode 100644
index 8b2e84b..0000000
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/bucket/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-/**
- * Package contains test classes for s3 bucket requests.
- */
-package org.apache.hadoop.ozone.om.request.s3.bucket;
\ No newline at end of file
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/security/TestOMDelegationTokenRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/security/TestOMDelegationTokenRequest.java
new file mode 100644
index 0000000..08c95f6
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/security/TestOMDelegationTokenRequest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.request.security;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OMMetrics;
+import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS;
+import org.junit.Rule;
+import org.junit.Before;
+import org.junit.After;
+import org.junit.rules.TemporaryFolder;
+import org.mockito.Mockito;
+import static org.mockito.Mockito.when;
+
+/**
+ * Base class for testing OM delegation token request.
+ */
+@SuppressWarnings("visibilitymodifier")
+public class TestOMDelegationTokenRequest {
+
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
+
+ protected OzoneManager ozoneManager;
+ protected OMMetrics omMetrics;
+ protected OMMetadataManager omMetadataManager;
+ protected ConfigurationSource conf;
+
+ // Just setting OzoneManagerDoubleBuffer which does nothing.
+ protected OzoneManagerDoubleBufferHelper ozoneManagerDoubleBufferHelper =
+ ((response, transactionIndex) -> {
+ return null;
+ });
+
+ @Before
+ public void setup() throws Exception {
+ ozoneManager = Mockito.mock(OzoneManager.class);
+
+ conf = new OzoneConfiguration();
+ ((OzoneConfiguration) conf)
+ .set(OZONE_OM_DB_DIRS, folder.newFolder().getAbsolutePath());
+ omMetadataManager = new OmMetadataManagerImpl((OzoneConfiguration) conf);
+ when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager);
+ }
+
+ @After
+ public void stop() {
+ Mockito.framework().clearInlineMocks();
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/security/TestOMGetDelegationTokenRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/security/TestOMGetDelegationTokenRequest.java
new file mode 100644
index 0000000..df0fcb9
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/security/TestOMGetDelegationTokenRequest.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.request.security;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import com.google.common.base.Optional;
+import java.util.UUID;
+import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import org.apache.hadoop.ozone.security.OzoneDelegationTokenSecretManager;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .OMRequest;
+import static org.apache.hadoop.ozone.security.OzoneTokenIdentifier.KIND_NAME;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.io.Text;
+import org.mockito.Mockito;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.when;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * The class tests OMGetDelegationTokenRequest.
+ */
+public class TestOMGetDelegationTokenRequest extends
+ TestOMDelegationTokenRequest {
+
+ private OzoneDelegationTokenSecretManager secretManager;
+ private OzoneTokenIdentifier identifier;
+ private Token<OzoneTokenIdentifier> token;
+ private Text tester;
+ private OMRequest originalRequest;
+ private OMRequest modifiedRequest;
+ private OMGetDelegationTokenRequest omGetDelegationTokenRequest;
+ final private String checkResponse = "";
+
+ @Before
+ public void setupGetDelegationToken() {
+ secretManager = Mockito.mock(OzoneDelegationTokenSecretManager.class);
+ when(ozoneManager.getDelegationTokenMgr()).thenReturn(secretManager);
+
+ setupToken();
+ setupRequest();
+ }
+
+ private void setupRequest() {
+ GetDelegationTokenRequestProto getDelegationTokenRequestProto =
+ GetDelegationTokenRequestProto.newBuilder()
+ .setRenewer(identifier.getRenewer().toString())
+ .build();
+
+ originalRequest = OMRequest.newBuilder()
+ .setClientId(UUID.randomUUID().toString())
+ .setCmdType(Type.GetDelegationToken)
+ .setGetDelegationTokenRequest(getDelegationTokenRequestProto)
+ .build();
+
+ omGetDelegationTokenRequest =
+ new OMGetDelegationTokenRequest(originalRequest);
+
+ modifiedRequest = null;
+ }
+
+ private void verifyUnchangedRequest() {
+ Assert.assertEquals(
+ originalRequest.getCmdType(),
+ modifiedRequest.getCmdType());
+ Assert.assertEquals(
+ originalRequest.getClientId(),
+ modifiedRequest.getClientId());
+ }
+
+ private void setupToken() {
+ tester = new Text("tester");
+ identifier = new OzoneTokenIdentifier(tester, tester, tester);
+ identifier.setOmCertSerialId("certID");
+ identifier.setOmServiceId("");
+
+ byte[] password = RandomStringUtils
+ .randomAlphabetic(10)
+ .getBytes(StandardCharsets.UTF_8);
+ Text service = new Text("OMTest:9862");
+ token = new Token<>(identifier.getBytes(), password, KIND_NAME, service);
+ }
+
+ private OMClientResponse setValidateAndUpdateCache() throws IOException {
+ modifiedRequest = omGetDelegationTokenRequest.preExecute(ozoneManager);
+ OMGetDelegationTokenRequest reqPreExecuted =
+ new OMGetDelegationTokenRequest(modifiedRequest);
+
+ long txLogIndex = 1L;
+ return reqPreExecuted.validateAndUpdateCache(
+ ozoneManager, txLogIndex, ozoneManagerDoubleBufferHelper);
+ }
+
+ @Test
+ public void testPreExecuteWithNonNullToken() throws Exception {
+ /* Let token of ozoneManager.getDelegationToken() is nonNull. */
+ when(ozoneManager.getDelegationToken(tester)).thenReturn(token);
+
+ long tokenRenewInterval = 1000L;
+ when(ozoneManager.getDelegationTokenMgr().getTokenRenewInterval())
+ .thenReturn(tokenRenewInterval);
+
+ modifiedRequest = omGetDelegationTokenRequest.preExecute(ozoneManager);
+ verifyUnchangedRequest();
+
+ long originalInterval = originalRequest.getUpdateGetDelegationTokenRequest()
+ .getTokenRenewInterval();
+ long renewInterval = modifiedRequest.getUpdateGetDelegationTokenRequest()
+ .getTokenRenewInterval();
+ Assert.assertNotEquals(originalInterval, renewInterval);
+ Assert.assertEquals(tokenRenewInterval, renewInterval);
+
+ /* In preExecute(), if the token is nonNull
+ we set GetDelegationTokenResponse with response. */
+ Assert.assertNotEquals(checkResponse,
+ modifiedRequest.getUpdateGetDelegationTokenRequest()
+ .getGetDelegationTokenResponse()
+ .toString());
+ Assert.assertNotNull(modifiedRequest
+ .getUpdateGetDelegationTokenRequest()
+ .getGetDelegationTokenResponse());
+ }
+
+ @Test
+ public void testPreExecuteWithNullToken() throws Exception {
+ /* Let token of ozoneManager.getDelegationToken() is Null. */
+ when(ozoneManager.getDelegationToken(tester)).thenReturn(null);
+
+ modifiedRequest = omGetDelegationTokenRequest.preExecute(ozoneManager);
+ verifyUnchangedRequest();
+
+ /* In preExecute(), if the token is null
+ we do not set GetDelegationTokenResponse with response. */
+ Assert.assertEquals(checkResponse,
+ modifiedRequest.getUpdateGetDelegationTokenRequest()
+ .getGetDelegationTokenResponse()
+ .toString());
+ }
+
+ @Test
+ public void testValidateAndUpdateCacheWithNonNullToken() throws Exception {
+ /* Let token of ozoneManager.getDelegationToken() is nonNull. */
+ when(ozoneManager.getDelegationToken(tester)).thenReturn(token);
+
+ /* Mock OzoneDelegationTokenSecretManager#updateToken() to
+ * get specific renewTime for verifying OMClientResponse returned by
+ * validateAndUpdateCache(). */
+ long renewTime = 1000L;
+ when(secretManager.updateToken(
+ any(Token.class), any(OzoneTokenIdentifier.class), anyLong()))
+ .thenReturn(renewTime);
+
+ OMClientResponse clientResponse = setValidateAndUpdateCache();
+
+ Optional<Long> responseRenewTime = Optional.fromNullable(
+ omMetadataManager.getDelegationTokenTable().get(identifier));
+ Assert.assertEquals(Optional.of(renewTime), responseRenewTime);
+
+ Assert.assertEquals(Status.OK, clientResponse.getOMResponse().getStatus());
+ }
+
+ @Test
+ public void testValidateAndUpdateCacheWithNullToken() throws Exception {
+ /* Let token of ozoneManager.getDelegationToken() is Null. */
+ when(ozoneManager.getDelegationToken(tester)).thenReturn(null);
+
+ OMClientResponse clientResponse = setValidateAndUpdateCache();
+
+ boolean hasResponse = modifiedRequest.getUpdateGetDelegationTokenRequest()
+ .getGetDelegationTokenResponse().hasResponse();
+ Assert.assertFalse(hasResponse);
+
+ Optional<Long> responseRenewTime = Optional.fromNullable(
+ omMetadataManager.getDelegationTokenTable().get(identifier));
+ Assert.assertEquals(Optional.absent(), responseRenewTime);
+
+ Assert.assertEquals(Status.OK, clientResponse.getOMResponse().getStatus());
+ }
+
+ @Test
+ public void testValidateAndUpdateCacheWithException() throws Exception {
+ /* Create a token that causes InvalidProtocolBufferException by
+ * OzoneTokenIdentifier#readProtoBuf(). */
+ Token<OzoneTokenIdentifier> exceptToken = new Token<>();
+ when(ozoneManager.getDelegationToken(tester)).thenReturn(exceptToken);
+
+ OMClientResponse clientResponse = setValidateAndUpdateCache();
+
+ boolean hasResponse = modifiedRequest.getUpdateGetDelegationTokenRequest()
+ .getGetDelegationTokenResponse().hasResponse();
+ Assert.assertTrue(hasResponse);
+
+ Assert.assertNotEquals(Status.OK,
+ clientResponse.getOMResponse().getStatus());
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/bucket/package-info.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/security/package-info.java
similarity index 87%
rename from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/bucket/package-info.java
rename to hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/security/package-info.java
index f484ecc..4a72a86 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/bucket/package-info.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/security/package-info.java
@@ -18,7 +18,6 @@
*/
/**
- * Package contains classes related to s3 bucket responses.
+ * Package contains test classes for delegation token requests.
*/
-package org.apache.hadoop.ozone.om.response.s3.bucket;
-
+package org.apache.hadoop.ozone.om.request.security;
\ No newline at end of file
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeCreateRequest.java
index 481304e..05d0eba 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeCreateRequest.java
@@ -20,9 +20,11 @@
import java.util.UUID;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
import org.apache.hadoop.ozone.om.response.volume.OMVolumeCreateResponse;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.LambdaTestUtils;
import org.junit.Assert;
import org.junit.Test;
@@ -49,6 +51,9 @@
String adminName = UUID.randomUUID().toString();
String ownerName = UUID.randomUUID().toString();
doPreExecute(volumeName, adminName, ownerName);
+ // Verify exception thrown on invalid volume name
+ LambdaTestUtils.intercept(OMException.class, "Invalid volume name: v1",
+ () -> doPreExecute("v1", adminName, ownerName));
}
@Test
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeSetOwnerRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeSetOwnerRequest.java
index 3c69a3b..c71c1a8 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeSetOwnerRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeSetOwnerRequest.java
@@ -18,6 +18,9 @@
package org.apache.hadoop.ozone.om.request.volume;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
import java.util.UUID;
import org.junit.Assert;
@@ -189,4 +192,50 @@
Assert.assertEquals(OzoneManagerProtocolProtos.Status.REPLAY,
omClientResponse.getOMResponse().getStatus());
}
+
+
+ @Test
+ public void testOwnSameVolumeTwice() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String owner = "user1";
+ TestOMRequestUtils.addVolumeToDB(volumeName, owner, omMetadataManager);
+ TestOMRequestUtils.addUserToDB(volumeName, owner, omMetadataManager);
+ String newOwner = "user2";
+
+ // Create request to set new owner
+ OMRequest omRequest =
+ TestOMRequestUtils.createSetVolumePropertyRequest(volumeName, newOwner);
+
+ OMVolumeSetOwnerRequest setOwnerRequest =
+ new OMVolumeSetOwnerRequest(omRequest);
+ // Execute the request
+ setOwnerRequest.preExecute(ozoneManager);
+ OMClientResponse omClientResponse = setOwnerRequest.validateAndUpdateCache(
+ ozoneManager, 1, ozoneManagerDoubleBufferHelper);
+ // Response status should be OK and success flag should be true.
+ Assert.assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ omClientResponse.getOMResponse().getStatus());
+ Assert.assertTrue(omClientResponse.getOMResponse().getSuccess());
+
+ // Execute the same request again but with higher index
+ setOwnerRequest.preExecute(ozoneManager);
+ omClientResponse = setOwnerRequest.validateAndUpdateCache(
+ ozoneManager, 2, ozoneManagerDoubleBufferHelper);
+ // Response status should be OK, but success flag should be false.
+ Assert.assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ omClientResponse.getOMResponse().getStatus());
+ Assert.assertFalse(omClientResponse.getOMResponse().getSuccess());
+
+ // Check volume names list
+ OzoneManagerProtocolProtos.UserVolumeInfo userVolumeInfo =
+ omMetadataManager.getUserTable().get(newOwner);
+ Assert.assertNotNull(userVolumeInfo);
+ List<String> volumeNamesList = userVolumeInfo.getVolumeNamesList();
+ Assert.assertEquals(1, volumeNamesList.size());
+
+ Set<String> volumeNamesSet = new HashSet<>(volumeNamesList);
+ // If the set size isn't equal to list size, there are duplicates
+ // in the list (which was the bug before the fix).
+ Assert.assertEquals(volumeNamesList.size(), volumeNamesSet.size());
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestOMResponseUtils.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestOMResponseUtils.java
index 5418fcc..01f8baa 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestOMResponseUtils.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestOMResponseUtils.java
@@ -20,12 +20,6 @@
package org.apache.hadoop.ozone.om.response;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
-import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
-import org.apache.hadoop.ozone.om.request.s3.bucket.S3BucketCreateRequest;
-import org.apache.hadoop.ozone.om.response.bucket.OMBucketCreateResponse;
-import org.apache.hadoop.ozone.om.response.s3.bucket.S3BucketCreateResponse;
-import org.apache.hadoop.ozone.om.response.volume.OMVolumeCreateResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.util.Time;
/**
@@ -44,41 +38,4 @@
}
- public static S3BucketCreateResponse createS3BucketResponse(String userName,
- String volumeName, String s3BucketName) {
- OzoneManagerProtocolProtos.OMResponse omResponse =
- OzoneManagerProtocolProtos.OMResponse.newBuilder()
- .setCmdType(OzoneManagerProtocolProtos.Type.CreateS3Bucket)
- .setStatus(OzoneManagerProtocolProtos.Status.OK)
- .setSuccess(true)
- .setCreateS3BucketResponse(
- OzoneManagerProtocolProtos.S3CreateBucketResponse
- .getDefaultInstance())
- .build();
-
- OzoneManagerProtocolProtos.UserVolumeInfo userVolumeInfo =
- OzoneManagerProtocolProtos.UserVolumeInfo.newBuilder()
- .setObjectID(1)
- .setUpdateID(1)
- .addVolumeNames(volumeName).build();
-
- OmVolumeArgs omVolumeArgs = OmVolumeArgs.newBuilder()
- .setOwnerName(userName).setAdminName(userName)
- .setVolume(volumeName).setCreationTime(Time.now()).build();
-
- OMVolumeCreateResponse omVolumeCreateResponse =
- new OMVolumeCreateResponse(omResponse, omVolumeArgs, userVolumeInfo);
-
-
- OmBucketInfo omBucketInfo = TestOMResponseUtils.createBucket(
- volumeName, s3BucketName);
- OMBucketCreateResponse omBucketCreateResponse =
- new OMBucketCreateResponse(omResponse, omBucketInfo);
-
- String s3Mapping = S3BucketCreateRequest.formatS3MappingName(volumeName,
- s3BucketName);
- return
- new S3BucketCreateResponse(omResponse, omVolumeCreateResponse,
- omBucketCreateResponse, s3BucketName, s3Mapping);
- }
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/bucket/TestS3BucketCreateResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/bucket/TestS3BucketCreateResponse.java
deleted file mode 100644
index f4a76e3..0000000
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/bucket/TestS3BucketCreateResponse.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.ozone.om.response.s3.bucket;
-
-import java.util.UUID;
-
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
-import org.apache.hadoop.ozone.om.OMMetadataManager;
-import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
-import org.apache.hadoop.ozone.om.request.s3.bucket.S3BucketCreateRequest;
-import org.apache.hadoop.ozone.om.response.TestOMResponseUtils;
-import org.apache.hadoop.hdds.utils.db.BatchOperation;
-
-/**
- * Class to test S3BucketCreateResponse.
- */
-public class TestS3BucketCreateResponse {
-
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
-
- private OMMetadataManager omMetadataManager;
- private BatchOperation batchOperation;
-
- @Before
- public void setup() throws Exception {
- OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
- ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS,
- folder.newFolder().getAbsolutePath());
- omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration);
- batchOperation = omMetadataManager.getStore().initBatchOperation();
- }
-
-
- @Test
- public void testAddToDBBatch() throws Exception {
- String userName = UUID.randomUUID().toString();
- String s3BucketName = UUID.randomUUID().toString();
- String volumeName = S3BucketCreateRequest.formatOzoneVolumeName(userName);
-
- S3BucketCreateResponse s3BucketCreateResponse =
- TestOMResponseUtils.createS3BucketResponse(userName, volumeName,
- s3BucketName);
-
- s3BucketCreateResponse.addToDBBatch(omMetadataManager, batchOperation);
-
- // Do manual commit and see whether addToBatch is successful or not.
- omMetadataManager.getStore().commitBatchOperation(batchOperation);
-
- Assert.assertNotNull(omMetadataManager.getS3Table().get(s3BucketName));
- Assert.assertEquals(s3BucketCreateResponse.getS3Mapping(),
- omMetadataManager.getS3Table().get(s3BucketName));
-
- Assert.assertEquals(1,
- omMetadataManager.countRowsInTable(omMetadataManager.getBucketTable()));
- Assert.assertEquals(1,
- omMetadataManager.countRowsInTable(omMetadataManager.getVolumeTable()));
-
- Assert.assertEquals(omMetadataManager.getVolumeKey(volumeName),
- omMetadataManager.getVolumeTable().iterator().next().getKey());
- Assert.assertNotNull(omMetadataManager.getBucketKey(volumeName,
- s3BucketName), omMetadataManager.getBucketTable().iterator().next()
- .getKey());
-
- }
-}
-
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/bucket/TestS3BucketDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/bucket/TestS3BucketDeleteResponse.java
deleted file mode 100644
index 2c7610a..0000000
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/bucket/TestS3BucketDeleteResponse.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.om.response.s3.bucket;
-
-import java.util.UUID;
-
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
-import org.apache.hadoop.ozone.om.OMMetadataManager;
-import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
-import org.apache.hadoop.ozone.om.request.s3.bucket.S3BucketCreateRequest;
-import org.apache.hadoop.ozone.om.response.TestOMResponseUtils;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
- .OMResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
- .S3DeleteBucketResponse;
-import org.apache.hadoop.hdds.utils.db.BatchOperation;
-
-
-
-/**
- * Tests S3BucketDeleteResponse.
- */
-public class TestS3BucketDeleteResponse {
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
-
- private OMMetadataManager omMetadataManager;
- private BatchOperation batchOperation;
-
- @Before
- public void setup() throws Exception {
- OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
- ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS,
- folder.newFolder().getAbsolutePath());
- omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration);
- batchOperation = omMetadataManager.getStore().initBatchOperation();
- }
-
- @Test
- public void testAddToDBBatch() throws Exception {
- String s3BucketName = UUID.randomUUID().toString();
- String userName = OzoneConsts.OZONE;
- String volumeName = S3BucketCreateRequest.formatOzoneVolumeName(userName);
- S3BucketCreateResponse s3BucketCreateResponse =
- TestOMResponseUtils.createS3BucketResponse(userName, volumeName,
- s3BucketName);
-
- s3BucketCreateResponse.addToDBBatch(omMetadataManager, batchOperation);
-
- OMResponse omResponse = OMResponse.newBuilder().setCmdType(
- OzoneManagerProtocolProtos.Type.DeleteS3Bucket).setStatus(
- OzoneManagerProtocolProtos.Status.OK).setSuccess(true)
- .setDeleteS3BucketResponse(S3DeleteBucketResponse.newBuilder()).build();
-
- S3BucketDeleteResponse s3BucketDeleteResponse =
- new S3BucketDeleteResponse(omResponse, s3BucketName, volumeName);
-
- s3BucketDeleteResponse.addToDBBatch(omMetadataManager, batchOperation);
-
- omMetadataManager.getStore().commitBatchOperation(batchOperation);
-
- // Check now s3 bucket exists or not.
- Assert.assertNull(omMetadataManager.getS3Table().get(s3BucketName));
- Assert.assertNull(omMetadataManager.getBucketTable().get(
- omMetadataManager.getBucketKey(volumeName, s3BucketName)));
- }
-}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/bucket/package-info.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/bucket/package-info.java
deleted file mode 100644
index 364396b..0000000
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/bucket/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-/**
- * Package contains test classes for s3 bucket responses.
- */
-package org.apache.hadoop.ozone.om.response.s3.bucket;
\ No newline at end of file
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/TestOMDelegationTokenResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/TestOMDelegationTokenResponse.java
new file mode 100644
index 0000000..816e696
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/TestOMDelegationTokenResponse.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.response.security;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.db.BatchOperation;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.rules.TemporaryFolder;
+import java.io.IOException;
+
+/** Base test class for delegation token response. */
+@SuppressWarnings("visibilitymodifier")
+public class TestOMDelegationTokenResponse {
+
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
+
+ protected ConfigurationSource conf;
+ protected OMMetadataManager omMetadataManager;
+ protected BatchOperation batchOperation;
+
+ @Before
+ public void setup() throws IOException {
+ conf = new OzoneConfiguration();
+ ((OzoneConfiguration) conf).set(OMConfigKeys.OZONE_OM_DB_DIRS,
+ folder.newFolder().getAbsolutePath());
+ omMetadataManager = new OmMetadataManagerImpl((OzoneConfiguration) conf);
+ batchOperation = omMetadataManager.getStore().initBatchOperation();
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/TestOMGetDelegationTokenResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/TestOMGetDelegationTokenResponse.java
new file mode 100644
index 0000000..df90d7e
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/TestOMGetDelegationTokenResponse.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.response.security;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.om.request.security.OMGetDelegationTokenRequest;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .OMRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+ .OMResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.UpdateGetDelegationTokenRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status;
+import java.io.IOException;
+import java.util.UUID;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/** The class tests OMGetDelegationTokenResponse. */
+public class TestOMGetDelegationTokenResponse extends
+ TestOMDelegationTokenResponse {
+
+ private OzoneTokenIdentifier identifier;
+ private UpdateGetDelegationTokenRequest updateGetDelegationTokenRequest;
+
+ @Before
+ public void setupGetDelegationToken() {
+ Text tester = new Text("tester");
+ identifier = new OzoneTokenIdentifier(tester, tester, tester);
+ identifier.setOmCertSerialId("certID");
+
+ GetDelegationTokenRequestProto getDelegationTokenRequestProto =
+ GetDelegationTokenRequestProto.newBuilder()
+ .setRenewer(identifier.getRenewer().toString())
+ .build();
+
+ OMRequest omRequest = OMRequest.newBuilder()
+ .setClientId(UUID.randomUUID().toString())
+ .setCmdType(Type.GetDelegationToken)
+ .setGetDelegationTokenRequest(getDelegationTokenRequestProto)
+ .build();
+
+ updateGetDelegationTokenRequest =
+ new OMGetDelegationTokenRequest(omRequest)
+ .getOmRequest()
+ .getUpdateGetDelegationTokenRequest();
+ }
+
+ @Test
+ public void testAddToDBBatch() throws IOException {
+ OMResponse omResponse = OMResponse.newBuilder()
+ .setCmdType(Type.GetDelegationToken)
+ .setStatus(Status.OK)
+ .setSuccess(true)
+ .setGetDelegationTokenResponse(
+ updateGetDelegationTokenRequest
+ .getGetDelegationTokenResponse())
+ .build();
+
+ long renewTime = 1000L;
+ OMGetDelegationTokenResponse getDelegationTokenResponse =
+ new OMGetDelegationTokenResponse(identifier, renewTime, omResponse);
+
+ getDelegationTokenResponse.addToDBBatch(omMetadataManager, batchOperation);
+ omMetadataManager.getStore().commitBatchOperation(batchOperation);
+
+ long rowNumInTable = 1;
+ long rowNumInTokenTable = omMetadataManager
+ .countRowsInTable(omMetadataManager.getDelegationTokenTable());
+ Assert.assertEquals(rowNumInTable, rowNumInTokenTable);
+
+ long renewTimeInTable = omMetadataManager.getDelegationTokenTable()
+ .get(identifier);
+ Assert.assertEquals(renewTime, renewTimeInTable);
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/package-info.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/package-info.java
similarity index 87%
copy from hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/package-info.java
copy to hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/package-info.java
index 80c1985..1c197c6 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/package-info.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/security/package-info.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.ozone.web.ozShell;
/**
- * Tests for ozone shell..
+ * Package contains test classes for delegation token responses.
*/
+package org.apache.hadoop.ozone.om.response.security;
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestAWSV4AuthValidator.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestAWSV4AuthValidator.java
similarity index 100%
rename from hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestAWSV4AuthValidator.java
rename to hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestAWSV4AuthValidator.java
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java
index 74a74ea..03c9732 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java
@@ -24,6 +24,7 @@
import org.apache.hadoop.hdds.security.x509.certificate.client.OMCertificateClient;
import org.apache.hadoop.hdds.server.ServerUtils;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
import org.apache.hadoop.ozone.om.S3SecretManager;
@@ -396,8 +397,15 @@
private OzoneDelegationTokenSecretManager
createSecretManager(OzoneConfiguration config, long tokenMaxLife,
long expiry, long tokenRemoverScanTime) throws IOException {
- return new OzoneDelegationTokenSecretManager(config, tokenMaxLife,
- expiry, tokenRemoverScanTime, serviceRpcAdd, s3SecretManager,
- certificateClient);
+ return new OzoneDelegationTokenSecretManager.Builder()
+ .setConf(config)
+ .setTokenMaxLifetime(tokenMaxLife)
+ .setTokenRenewInterval(expiry)
+ .setTokenRemoverScanInterval(tokenRemoverScanTime)
+ .setService(serviceRpcAdd)
+ .setS3SecretManager(s3SecretManager)
+ .setCertificateClient(certificateClient)
+ .setOmServiceId(OzoneConsts.OM_SERVICE_ID_DEFAULT)
+ .build();
}
}
\ No newline at end of file
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneTokenIdentifier.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneTokenIdentifier.java
index f26869d..518953f 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneTokenIdentifier.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneTokenIdentifier.java
@@ -17,6 +17,10 @@
*/
package org.apache.hadoop.ozone.security;
+import javax.crypto.KeyGenerator;
+import javax.crypto.Mac;
+import javax.crypto.SecretKey;
+import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
@@ -38,18 +42,19 @@
import java.util.Collections;
import java.util.List;
import java.util.Map;
-import javax.crypto.KeyGenerator;
-import javax.crypto.Mac;
-import javax.crypto.SecretKey;
-import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.conf.Configuration;
+
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
import org.apache.hadoop.security.ssl.TestSSLFactory;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.RandomUtils;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -86,10 +91,10 @@
base.mkdirs();
}
- private Configuration createConfiguration(boolean clientCert,
+ private ConfigurationSource createConfiguration(boolean clientCert,
boolean trustStore)
throws Exception {
- Configuration conf = new Configuration();
+ OzoneConfiguration conf = new OzoneConfiguration();
KeyStoreTestUtil.setupSSLConfig(KEYSTORES_DIR, sslConfsDir, conf,
clientCert, trustStore, EXCLUDE_CIPHERS);
sslConfsDir = KeyStoreTestUtil.getClasspathDir(TestSSLFactory.class);
@@ -302,4 +307,24 @@
id.setOmCertSerialId("123");
return id;
}
+
+ @Test
+ public void testTokenSerialization() throws IOException {
+ OzoneTokenIdentifier idEncode = getIdentifierInst();
+ idEncode.setOmServiceId("defaultServiceId");
+ Token<OzoneTokenIdentifier> token = new Token<OzoneTokenIdentifier>(
+ idEncode.getBytes(), new byte[0], new Text("OzoneToken"),
+ new Text("om1:9862,om2:9852,om3:9852"));
+ String encodedStr = token.encodeToUrlString();
+
+ Token<OzoneTokenIdentifier> tokenDecode = new Token<>();
+ tokenDecode.decodeFromUrlString(encodedStr);
+
+ ByteArrayInputStream buf = new ByteArrayInputStream(
+ tokenDecode.getIdentifier());
+ DataInputStream in = new DataInputStream(buf);
+ OzoneTokenIdentifier idDecode = new OzoneTokenIdentifier();
+ idDecode.readFields(in);
+ Assert.assertEquals(idEncode, idDecode);
+ }
}
\ No newline at end of file
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestOzoneAdministrators.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestOzoneAdministrators.java
new file mode 100644
index 0000000..d876ca1
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestOzoneAdministrators.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.security.acl;
+
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD;
+
+/**
+ * Test Ozone Administrators from OzoneNativeAuthorizer.
+ */
+public class TestOzoneAdministrators {
+
+ private static OzoneNativeAuthorizer nativeAuthorizer;
+
+ @BeforeClass
+ public static void setup() {
+ nativeAuthorizer = new OzoneNativeAuthorizer();
+ }
+
+ @Test
+ public void testCreateVolume() throws Exception {
+ OzoneObj obj = getTestVolumeobj("testvolume");
+ RequestContext context = getUserRequestContext("testuser",
+ IAccessAuthorizer.ACLType.CREATE);
+ testAdminOperations(obj, context);
+ }
+
+ @Test
+ public void testListAllVolume() throws Exception {
+ OzoneObj obj = getTestVolumeobj("/");
+ RequestContext context = getUserRequestContext("testuser",
+ IAccessAuthorizer.ACLType.LIST);
+ testAdminOperations(obj, context);
+ }
+
+ private void testAdminOperations(OzoneObj obj, RequestContext context)
+ throws OMException {
+ nativeAuthorizer.setOzoneAdmins(Collections.emptyList());
+ Assert.assertFalse("empty admin list disallow anyone to perform " +
+ "admin operations", nativeAuthorizer.checkAccess(obj, context));
+
+ nativeAuthorizer.setOzoneAdmins(
+ Collections.singletonList(OZONE_ADMINISTRATORS_WILDCARD));
+ Assert.assertTrue("wildcard admin allows everyone to perform admin" +
+ " operations", nativeAuthorizer.checkAccess(obj, context));
+
+ nativeAuthorizer.setOzoneAdmins(
+ Collections.singletonList("testuser"));
+ Assert.assertTrue("matching admins are allowed to perform admin " +
+ "operations", nativeAuthorizer.checkAccess(obj, context));
+
+ nativeAuthorizer.setOzoneAdmins(
+ Arrays.asList(new String[]{"testuser2", "testuser"}));
+ Assert.assertTrue("matching admins are allowed to perform admin " +
+ "operations", nativeAuthorizer.checkAccess(obj, context));
+
+ nativeAuthorizer.setOzoneAdmins(
+ Arrays.asList(new String[]{"testuser2", "testuser3"}));
+ Assert.assertFalse("mismatching admins are not allowed perform " +
+ "admin operations", nativeAuthorizer.checkAccess(obj, context));
+ }
+
+ private RequestContext getUserRequestContext(String username,
+ IAccessAuthorizer.ACLType type) {
+ return RequestContext.newBuilder()
+ .setClientUgi(UserGroupInformation.createRemoteUser(username))
+ .setAclType(IAccessAuthorizer.ACLIdentityType.USER)
+ .setAclRights(type)
+ .build();
+ }
+
+ private OzoneObj getTestVolumeobj(String volumename) {
+ return OzoneObjInfo.Builder.newBuilder()
+ .setResType(OzoneObj.ResourceType.VOLUME)
+ .setStoreType(OzoneObj.StoreType.OZONE)
+ .setVolumeName(volumename).build();
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestOzoneNativeAuthorizer.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestOzoneNativeAuthorizer.java
index ecbe7a3..ac1ad20 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestOzoneNativeAuthorizer.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestOzoneNativeAuthorizer.java
@@ -62,7 +62,6 @@
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS_NATIVE;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType.ANONYMOUS;
import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType.GROUP;
@@ -102,14 +101,13 @@
private static PrefixManager prefixManager;
private static OMMetadataManager metadataManager;
private static OzoneNativeAuthorizer nativeAuthorizer;
-
- private static UserGroupInformation ugi;
+ private static UserGroupInformation adminUgi;
+ private static UserGroupInformation testUgi;
private static OzoneObj volObj;
private static OzoneObj buckObj;
private static OzoneObj keyObj;
private static OzoneObj prefixObj;
- private static long keySessionId;
@Parameterized.Parameters
public static Collection<Object[]> data() {
@@ -149,7 +147,7 @@
OZONE_ACL_AUTHORIZER_CLASS_NATIVE);
File dir = GenericTestUtils.getRandomizedTestDir();
ozConfig.set(OZONE_METADATA_DIRS, dir.toString());
- ozConfig.set(OZONE_ADMINISTRATORS, OZONE_ADMINISTRATORS_WILDCARD);
+ ozConfig.set(OZONE_ADMINISTRATORS, "om");
metadataManager = new OmMetadataManagerImpl(ozConfig);
volumeManager = new VolumeManagerImpl(metadataManager, ozConfig);
@@ -160,9 +158,12 @@
metadataManager, ozConfig, "om1", null);
nativeAuthorizer = new OzoneNativeAuthorizer(volumeManager, bucketManager,
- keyManager, prefixManager);
- //keySession.
- ugi = UserGroupInformation.getCurrentUser();
+ keyManager, prefixManager,
+ Collections.singletonList("om"));
+ adminUgi = UserGroupInformation.createUserForTesting("om",
+ new String[]{"ozone"});
+ testUgi = UserGroupInformation.createUserForTesting("testuser",
+ new String[]{"test"});
}
private void createKey(String volume,
@@ -174,8 +175,8 @@
.setFactor(HddsProtos.ReplicationFactor.ONE)
.setDataSize(0)
.setType(HddsProtos.ReplicationType.STAND_ALONE)
- .setAcls(OzoneAclUtil.getAclList(ugi.getUserName(), ugi.getGroups(),
- ALL, ALL))
+ .setAcls(OzoneAclUtil.getAclList(testUgi.getUserName(),
+ testUgi.getGroups(), ALL, ALL))
.build();
if (keyName.split(OZONE_URI_DELIMITER).length > 1) {
@@ -187,7 +188,6 @@
keySession.getKeyInfo().getLatestVersionLocations()
.getLocationList());
keyManager.commitKey(keyArgs, keySession.getId());
- keySessionId = keySession.getId();
}
keyObj = new OzoneObjInfo.Builder()
@@ -217,8 +217,8 @@
private void createVolume(String volumeName) throws IOException {
OmVolumeArgs volumeArgs = OmVolumeArgs.newBuilder()
.setVolume(volumeName)
- .setAdminName("bilbo")
- .setOwnerName("bilbo")
+ .setAdminName(adminUgi.getUserName())
+ .setOwnerName(testUgi.getUserName())
.build();
TestOMRequestUtils.addVolumeToOM(metadataManager, volumeArgs);
volObj = new OzoneObjInfo.Builder()
@@ -240,10 +240,10 @@
@Test
public void testCheckAccessForBucket() throws Exception {
- OzoneAcl userAcl = new OzoneAcl(USER, ugi.getUserName(), parentDirUserAcl,
- ACCESS);
- OzoneAcl groupAcl = new OzoneAcl(GROUP, ugi.getGroups().size() > 0 ?
- ugi.getGroups().get(0) : "", parentDirGroupAcl, ACCESS);
+ OzoneAcl userAcl = new OzoneAcl(USER, testUgi.getUserName(),
+ parentDirUserAcl, ACCESS);
+ OzoneAcl groupAcl = new OzoneAcl(GROUP, testUgi.getGroups().size() > 0 ?
+ testUgi.getGroups().get(0) : "", parentDirGroupAcl, ACCESS);
// Set access for volume.
// We should directly add to table because old API's update to DB.
@@ -258,10 +258,10 @@
@Test
public void testCheckAccessForKey() throws Exception {
- OzoneAcl userAcl = new OzoneAcl(USER, ugi.getUserName(), parentDirUserAcl,
- ACCESS);
- OzoneAcl groupAcl = new OzoneAcl(GROUP, ugi.getGroups().size() > 0 ?
- ugi.getGroups().get(0) : "", parentDirGroupAcl, ACCESS);
+ OzoneAcl userAcl = new OzoneAcl(USER, testUgi.getUserName(),
+ parentDirUserAcl, ACCESS);
+ OzoneAcl groupAcl = new OzoneAcl(GROUP, testUgi.getGroups().size() > 0 ?
+ testUgi.getGroups().get(0) : "", parentDirGroupAcl, ACCESS);
// Set access for volume & bucket. We should directly add to table
// because old API's update to DB.
@@ -284,10 +284,10 @@
.setStoreType(OZONE)
.build();
- OzoneAcl userAcl = new OzoneAcl(USER, ugi.getUserName(), parentDirUserAcl,
- ACCESS);
- OzoneAcl groupAcl = new OzoneAcl(GROUP, ugi.getGroups().size() > 0 ?
- ugi.getGroups().get(0) : "", parentDirGroupAcl, ACCESS);
+ OzoneAcl userAcl = new OzoneAcl(USER, testUgi.getUserName(),
+ parentDirUserAcl, ACCESS);
+ OzoneAcl groupAcl = new OzoneAcl(GROUP, testUgi.getGroups().size() > 0 ?
+ testUgi.getGroups().get(0) : "", parentDirGroupAcl, ACCESS);
// Set access for volume & bucket. We should directly add to table
// because old API's update to DB.
@@ -348,18 +348,13 @@
private void resetAclsAndValidateAccess(OzoneObj obj,
ACLIdentityType accessType, IOzoneAcl aclImplementor)
throws IOException {
-
List<OzoneAcl> acls;
- String user = "";
- String group = "";
-
- user = ugi.getUserName();
- if (ugi.getGroups().size() > 0) {
- group = ugi.getGroups().get(0);
- }
+ String user = testUgi.getUserName();
+ String group = (testUgi.getGroups().size() > 0) ?
+ testUgi.getGroups().get(0): "";
RequestContext.Builder builder = new RequestContext.Builder()
- .setClientUgi(ugi)
+ .setClientUgi(testUgi)
.setAclType(accessType);
// Get all acls.
@@ -402,11 +397,16 @@
validateNone(obj, builder);
continue;
}
- assertEquals("Acl to check:" + a1 + " accessType:" +
- accessType + " path:" + obj.getPath(),
- expectedAclResult, nativeAuthorizer.checkAccess(obj,
- builder.setAclRights(a1).build()));
-
+ String msg = "Acl to check:" + a1 + " accessType:" +
+ accessType + " path:" + obj.getPath();
+ if (a1.equals(CREATE) && obj.getResourceType().equals(VOLUME)) {
+ assertEquals(msg, nativeAuthorizer.getOzoneAdmins().contains(user),
+ nativeAuthorizer.checkAccess(obj,
+ builder.setAclRights(a1).build()));
+ } else {
+ assertEquals(msg, expectedAclResult, nativeAuthorizer.checkAccess(obj,
+ builder.setAclRights(a1).build()));
+ }
List<ACLType> aclsToBeValidated =
Arrays.stream(ACLType.values()).collect(Collectors.toList());
List<ACLType> aclsToBeAdded =
@@ -499,10 +499,10 @@
private String getAclName(ACLIdentityType identityType) {
switch (identityType) {
case USER:
- return ugi.getUserName();
+ return testUgi.getUserName();
case GROUP:
- if (ugi.getGroups().size() > 0) {
- return ugi.getGroups().get(0);
+ if (testUgi.getGroups().size() > 0) {
+ return testUgi.getGroups().get(0);
}
default:
return "";
@@ -519,10 +519,15 @@
List<ACLType> allAcls = new ArrayList<>(Arrays.asList(ACLType.values()));
allAcls.remove(ALL);
allAcls.remove(NONE);
+ RequestContext ctx = builder.build();
+ boolean expectedResult = expectedAclResult;
+ if (nativeAuthorizer.getOzoneAdmins().contains(
+ ctx.getClientUgi().getUserName())) {
+ expectedResult = true;
+ }
for (ACLType a : allAcls) {
- assertEquals("User should have right " + a + ".",
- nativeAuthorizer.checkAccess(obj,
- builder.setAclRights(a).build()), expectedAclResult);
+ assertEquals("User should have right " + a + ".",
+ expectedResult, nativeAuthorizer.checkAccess(obj, ctx));
}
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/TestObjectPrinter.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/TestObjectPrinter.java
deleted file mode 100644
index 23737f0..0000000
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/TestObjectPrinter.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.ozShell;
-
-import java.io.IOException;
-import java.time.Instant;
-import java.util.ArrayList;
-
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-/**
- * Test the json object printer.
- */
-public class TestObjectPrinter {
-
- @Test
- public void printObjectAsJson() throws IOException {
-
- OzoneConfiguration conf = new OzoneConfiguration();
- OzoneVolume volume =
- new OzoneVolume(conf, Mockito.mock(ClientProtocol.class), "name",
- "admin", "owner", 1L, Instant.EPOCH.toEpochMilli(),
- new ArrayList<>());
-
- String result = ObjectPrinter.getObjectAsJson(volume);
- Assert.assertTrue("Result is not a proper json",
- result.contains("\"owner\""));
- Assert.assertTrue("Result is not a proper json",
- result.contains("\"1970-01-01T00:00:00Z\""));
- }
-}
\ No newline at end of file
diff --git a/hadoop-ozone/ozonefs-lib-current/pom.xml b/hadoop-ozone/ozonefs-lib-current/pom.xml
index 425f2d2..697596c 100644
--- a/hadoop-ozone/ozonefs-lib-current/pom.xml
+++ b/hadoop-ozone/ozonefs-lib-current/pom.xml
@@ -113,11 +113,6 @@
<include>org.apache.commons.validator.**.*</include>
<include>org.sqlite.**.*</include>
<include>org.apache.thrift.**.*</include>
- <!-- level db -->
- <include>org.iq80.**.*</include>
- <include>org.fusesource.**.*</include>
- <!-- http client and core -->
- <include>org.apache.http.**.*</include>
</includes>
</relocation>
<relocation>
@@ -186,6 +181,14 @@
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-server-framework</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-container-service</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
</exclusion>
<exclusion>
diff --git a/hadoop-ozone/ozonefs/pom.xml b/hadoop-ozone/ozonefs/pom.xml
index dca9915..8cc2434 100644
--- a/hadoop-ozone/ozonefs/pom.xml
+++ b/hadoop-ozone/ozonefs/pom.xml
@@ -78,13 +78,12 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>compile</scope>
+ <artifactId>hadoop-hdds-hadoop-dependency-client</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <scope>compile</scope>
+ <artifactId>hadoop-hdds-test-utils</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -92,6 +91,10 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-server-scm</artifactId>
<scope>test</scope>
</dependency>
@@ -110,30 +113,13 @@
<artifactId>hadoop-hdds-container-service</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdds-client</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-ozone-common</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- </dependency>
- <dependency>
+ <dependency>
<groupId>com.github.spotbugs</groupId>
<artifactId>spotbugs</artifactId>
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-client</artifactId>
@@ -148,7 +134,7 @@
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
- <version>1.10.19</version>
+ <version>${mockito1-powermock.version}</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -159,13 +145,12 @@
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-module-junit4</artifactId>
- <version>1.6.5</version>
+ <version>${powermock1.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-api-mockito</artifactId>
- <version>1.6.5</version>
<scope>test</scope>
</dependency>
</dependencies>
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzFs.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzFs.java
index 935826e..acab6d1 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzFs.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzFs.java
@@ -18,16 +18,16 @@
package org.apache.hadoop.fs.ozone;
-import org.apache.hadoop.hdds.annotation.InterfaceAudience;
-import org.apache.hadoop.hdds.annotation.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.DelegateToFileSystem;
-import org.apache.hadoop.ozone.OzoneConsts;
-
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.DelegateToFileSystem;
+import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.annotation.InterfaceStability;
+import org.apache.hadoop.ozone.OzoneConsts;
+
/**
* ozone implementation of AbstractFileSystem.
* This impl delegates to the OzoneFileSystem
@@ -42,4 +42,8 @@
OzoneConsts.OZONE_URI_SCHEME, false);
}
+ @Override
+ public int getUriDefaultPort() {
+ return -1;
+ }
}
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
index 2e7e9ae..f388107 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
@@ -25,16 +25,15 @@
import java.util.Iterator;
import java.util.List;
-import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileAlreadyExistsException;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
@@ -57,7 +56,9 @@
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenRenewer;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -112,7 +113,7 @@
}
public BasicOzoneClientAdapterImpl(String omHost, int omPort,
- Configuration hadoopConf, String volumeStr, String bucketStr)
+ ConfigurationSource hadoopConf, String volumeStr, String bucketStr)
throws IOException {
ClassLoader contextClassLoader =
@@ -121,7 +122,6 @@
try {
OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
-
if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
// When the host name or service id isn't given
// but ozone.om.service.ids is defined, declare failure.
@@ -294,8 +294,7 @@
try {
incrementCounter(Statistic.OBJECTS_QUERY);
OzoneFileStatus status = bucket.getFileStatus(key);
- makeQualified(status, uri, qualifiedPath, userName);
- return toFileStatusAdapter(status);
+ return toFileStatusAdapter(status, userName, uri, qualifiedPath);
} catch (OMException e) {
if (e.getResult() == OMException.ResultCodes.FILE_NOT_FOUND) {
@@ -306,15 +305,6 @@
}
}
- public void makeQualified(FileStatus status, URI uri, Path path,
- String username) {
- if (status instanceof OzoneFileStatus) {
- ((OzoneFileStatus) status)
- .makeQualified(uri, path,
- username, username);
- }
-
- }
@Override
public Iterator<BasicKeyInfo> listKeys(String pathKey) {
@@ -332,9 +322,7 @@
List<FileStatusAdapter> result = new ArrayList<>();
for (OzoneFileStatus status : statuses) {
- Path qualifiedPath = status.getPath().makeQualified(uri, workingDir);
- makeQualified(status, uri, qualifiedPath, username);
- result.add(toFileStatusAdapter(status));
+ result.add(toFileStatusAdapter(status, username, uri, workingDir));
}
return result;
} catch (OMException e) {
@@ -404,8 +392,10 @@
throws IOException, InterruptedException {
Token<OzoneTokenIdentifier> ozoneDt =
(Token<OzoneTokenIdentifier>) token;
+
OzoneClient ozoneClient =
- OzoneClientFactory.getRpcClient(conf);
+ OzoneClientFactory.getOzoneClient(OzoneConfiguration.of(conf),
+ ozoneDt);
return ozoneClient.getObjectStore().renewDelegationToken(ozoneDt);
}
@@ -415,7 +405,8 @@
Token<OzoneTokenIdentifier> ozoneDt =
(Token<OzoneTokenIdentifier>) token;
OzoneClient ozoneClient =
- OzoneClientFactory.getRpcClient(conf);
+ OzoneClientFactory.getOzoneClient(OzoneConfiguration.of(conf),
+ ozoneDt);
ozoneClient.getObjectStore().cancelDelegationToken(ozoneDt);
}
}
@@ -451,19 +442,23 @@
}
}
- private FileStatusAdapter toFileStatusAdapter(OzoneFileStatus status) {
+ private FileStatusAdapter toFileStatusAdapter(OzoneFileStatus status,
+ String owner, URI defaultUri, Path workingDir) {
+ OmKeyInfo keyInfo = status.getKeyInfo();
+ short replication = (short) keyInfo.getFactor().getNumber();
return new FileStatusAdapter(
- status.getLen(),
- status.getPath(),
+ keyInfo.getDataSize(),
+ new Path(OZONE_URI_DELIMITER + keyInfo.getKeyName())
+ .makeQualified(defaultUri, workingDir),
status.isDirectory(),
- status.getReplication(),
+ replication,
status.getBlockSize(),
- status.getModificationTime(),
- status.getAccessTime(),
- status.getPermission().toShort(),
- status.getOwner(),
- status.getGroup(),
- status.getPath(),
+ keyInfo.getModificationTime(),
+ keyInfo.getModificationTime(),
+ status.isDirectory() ? (short) 00777 : (short) 00666,
+ owner,
+ owner,
+ null,
getBlockLocations(status)
);
}
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
index dbd5f91..b7323ac 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
@@ -32,8 +32,6 @@
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import org.apache.hadoop.hdds.annotation.InterfaceAudience;
-import org.apache.hadoop.hdds.annotation.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.CreateFlag;
@@ -46,6 +44,11 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.annotation.InterfaceStability;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
@@ -57,7 +60,6 @@
import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME;
-
import org.apache.http.client.utils.URIBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -155,7 +157,15 @@
boolean isolatedClassloader =
conf.getBoolean("ozone.fs.isolated-classloader", defaultValue);
- this.adapter = createAdapter(conf, bucketStr, volumeStr, omHost, omPort,
+ ConfigurationSource source;
+ if (conf instanceof OzoneConfiguration) {
+ source = (ConfigurationSource) conf;
+ } else {
+ source = new LegacyHadoopConfigurationSource(conf);
+ }
+ this.adapter =
+ createAdapter(source, bucketStr,
+ volumeStr, omHost, omPort,
isolatedClassloader);
try {
@@ -174,7 +184,7 @@
}
}
- protected OzoneClientAdapter createAdapter(Configuration conf,
+ protected OzoneClientAdapter createAdapter(ConfigurationSource conf,
String bucketStr,
String volumeStr, String omHost, int omPort,
boolean isolatedClassloader) throws IOException {
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/FileStatusAdapter.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/FileStatusAdapter.java
index 64e43f5..56ba001 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/FileStatusAdapter.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/FileStatusAdapter.java
@@ -66,6 +66,7 @@
this.blockLocations = locations.clone();
}
+
public Path getPath() {
return path;
}
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/O3fsDtFetcher.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/O3fsDtFetcher.java
index a0ec01f..5ab3a00 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/O3fsDtFetcher.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/O3fsDtFetcher.java
@@ -21,18 +21,18 @@
import java.io.IOException;
import java.net.URI;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.DtFetcher;
import org.apache.hadoop.security.token.Token;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* A DT fetcher for OzoneFileSystem.
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java
index a6dd3a4..346b994 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java
@@ -18,16 +18,16 @@
package org.apache.hadoop.fs.ozone;
-import org.apache.hadoop.hdds.annotation.InterfaceAudience;
-import org.apache.hadoop.hdds.annotation.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.DelegateToFileSystem;
-import org.apache.hadoop.ozone.OzoneConsts;
-
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.DelegateToFileSystem;
+import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.annotation.InterfaceStability;
+import org.apache.hadoop.ozone.OzoneConsts;
+
/**
* ozone implementation of AbstractFileSystem.
* This impl delegates to the OzoneFileSystem
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapterImpl.java
index 975bbf7..d19f570 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapterImpl.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapterImpl.java
@@ -18,7 +18,8 @@
package org.apache.hadoop.fs.ozone;
import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
/**
@@ -44,7 +45,7 @@
}
public OzoneClientAdapterImpl(String omHost, int omPort,
- Configuration hadoopConf, String volumeStr, String bucketStr,
+ ConfigurationSource hadoopConf, String volumeStr, String bucketStr,
OzoneFSStorageStatistics storageStatistics)
throws IOException {
super(omHost, omPort, hadoopConf, volumeStr, bucketStr);
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
index a81de2c..20dd72f 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
@@ -24,7 +24,7 @@
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.annotation.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderTokenIssuer;
import org.apache.hadoop.fs.FileSystem;
@@ -36,9 +36,9 @@
* The Ozone Filesystem implementation.
* <p>
* This subclass is marked as private as code should not be creating it
- * directly; use {@link FileSystem#get(Configuration)} and variants to create
- * one. If cast to {@link OzoneFileSystem}, extra methods and features may be
- * accessed. Consider those private and unstable.
+ * directly; use {@link FileSystem#get(org.apache.hadoop.conf.Configuration)}
+ * and variants to create one. If cast to {@link OzoneFileSystem}, extra
+ * methods and features may be accessed. Consider those private and unstable.
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
@@ -85,7 +85,7 @@
}
@Override
- protected OzoneClientAdapter createAdapter(Configuration conf,
+ protected OzoneClientAdapter createAdapter(ConfigurationSource conf,
String bucketStr,
String volumeStr, String omHost, int omPort,
boolean isolatedClassloader) throws IOException {
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFsShell.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFsShell.java
index b36d368..3169ecd 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFsShell.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFsShell.java
@@ -17,11 +17,12 @@
*/
package org.apache.hadoop.fs.ozone;
-import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.shell.CommandFactory;
import org.apache.hadoop.fs.shell.FsCommand;
+import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.util.ToolRunner;
/** Provide command line access to a Ozone FileSystem. */
@@ -45,7 +46,7 @@
* Commands can be executed via {@link #run(String[])}
* @param conf the hadoop configuration
*/
- public OzoneFsShell(Configuration conf) {
+ public OzoneFsShell(OzoneConfiguration conf) {
super(conf);
}
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/PosixOzoneFileSystem.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/PosixOzoneFileSystem.java
new file mode 100644
index 0000000..b7f4d8b
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/PosixOzoneFileSystem.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.Progressable;
+import java.io.IOException;
+
+/**
+ * The PosixOzoneFileSystem implementation.
+ * The create function in this class overrides
+ * the create function of OzoneFileSystem class.
+ * This create function first creates a file, then
+ * closes it and again calls the create function
+ * of super class.
+ */
+
+public class PosixOzoneFileSystem extends OzoneFileSystem {
+ @Override
+ public FSDataOutputStream create(Path f, FsPermission permission,
+ boolean overwrite, int bufferSize,
+ short replication, long blockSize,
+ Progressable progress) throws IOException {
+ FSDataOutputStream cstream = super.create(f, permission, overwrite,
+ bufferSize, replication, blockSize, progress);
+ cstream.close();
+ return super.create(f, permission, overwrite, bufferSize,
+ replication, blockSize, progress);
+ }
+}
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithMocks.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithMocks.java
index 2d55812..d204ad5 100644
--- a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithMocks.java
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithMocks.java
@@ -17,18 +17,13 @@
*/
package org.apache.hadoop.fs.ozone;
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.OzoneConfigKeys;
@@ -38,8 +33,14 @@
import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.security.UserGroupInformation;
+
+import static org.junit.Assert.assertEquals;
import org.junit.Test;
import org.junit.runner.RunWith;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
@@ -55,7 +56,7 @@
@Test
public void testFSUriWithHostPortOverrides() throws Exception {
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
mockClientFactory(conf, 5899);
mockUser();
@@ -71,7 +72,7 @@
@Test
public void testFSUriWithHostPortUnspecified() throws Exception {
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
final int omPort = OmUtils.getOmRpcPort(conf);
mockClientFactory(conf, omPort);
mockUser();
@@ -90,7 +91,7 @@
@Test
public void testFSUriHostVersionDefault() throws Exception {
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
mockClientFactory(conf);
mockUser();
@@ -106,7 +107,7 @@
@Test
public void testReplicationDefaultValue()
throws IOException, URISyntaxException {
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
int defaultValue = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION, 3);
mockClientFactory(conf);
mockUser();
@@ -120,7 +121,7 @@
@Test
public void testReplicationCustomValue()
throws IOException, URISyntaxException {
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
short configured = 1;
conf.setInt(OzoneConfigKeys.OZONE_REPLICATION, configured);
mockClientFactory(conf);
@@ -144,20 +145,20 @@
return ozoneClient;
}
- private void mockClientFactory(Configuration conf, int omPort)
+ private void mockClientFactory(ConfigurationSource conf, int omPort)
throws IOException {
OzoneClient ozoneClient = mockClient();
PowerMockito.mockStatic(OzoneClientFactory.class);
PowerMockito.when(OzoneClientFactory.getRpcClient(eq("local.host"),
- eq(omPort), eq(conf))).thenReturn(ozoneClient);
+ eq(omPort), any())).thenReturn(ozoneClient);
}
- private void mockClientFactory(Configuration conf) throws IOException {
+ private void mockClientFactory(ConfigurationSource conf) throws IOException {
OzoneClient ozoneClient = mockClient();
PowerMockito.mockStatic(OzoneClientFactory.class);
- PowerMockito.when(OzoneClientFactory.getRpcClient(eq(conf)))
+ PowerMockito.when(OzoneClientFactory.getRpcClient(any()))
.thenReturn(ozoneClient);
}
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestReadWriteStatistics.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestReadWriteStatistics.java
index b8f8507..3659cdc 100644
--- a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestReadWriteStatistics.java
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestReadWriteStatistics.java
@@ -18,20 +18,6 @@
package org.apache.hadoop.fs.ozone;
-import static org.mockito.Mockito.*;
-import static org.junit.Assert.*;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CreateFlag;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.GlobalStorageStatistics;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.StorageStatistics;
-import org.junit.Before;
-import org.junit.Test;
-
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -42,6 +28,31 @@
import java.util.Arrays;
import java.util.EnumSet;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.GlobalStorageStatistics;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StorageStatistics;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import org.junit.Before;
+import org.junit.Test;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyBoolean;
+import static org.mockito.Mockito.anyInt;
+import static org.mockito.Mockito.anyShort;
+import static org.mockito.Mockito.anyString;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
/**
* Tests to check if bytes read and written and corresponding read and write
* operation counts are accounted properly in FileSystem statistics, when the
@@ -404,14 +415,14 @@
}
private void setupFileSystemToUseFakeClientAdapter() throws IOException {
- doReturn(fakeAdapter).when(fs).createAdapter(any(Configuration.class),
+ doReturn(fakeAdapter).when(fs).createAdapter(any(ConfigurationSource.class),
anyString(), anyString(), anyString(), anyInt(), anyBoolean());
}
private void initializeFS() throws IOException, URISyntaxException {
FileSystem.getGlobalStorageStatistics().reset();
URI fsUri = new URI("o3fs://volume.bucket.localhost");
- Configuration conf = new Configuration();
+ OzoneConfiguration conf = new OzoneConfiguration();
fs.initialize(fsUri, conf);
}
diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml
index 522e820..69c7e0e 100644
--- a/hadoop-ozone/pom.xml
+++ b/hadoop-ozone/pom.xml
@@ -26,6 +26,8 @@
<properties>
<docker.image>apache/ozone:${project.version}</docker.image>
+ <spring.version>5.2.5.RELEASE</spring.version>
+ <jooq.version>3.11.10</jooq.version>
</properties>
<modules>
<module>common</module>
@@ -110,6 +112,28 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-test-utils</artifactId>
+ <version>${hdds.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-hadoop-dependency-client</artifactId>
+ <version>${hdds.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-hadoop-dependency-server</artifactId>
+ <version>${hdds.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-hadoop-dependency-test</artifactId>
+ <version>${hdds.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-integration-test</artifactId>
<version>${ozone.version}</version>
<type>test-jar</type>
@@ -183,35 +207,10 @@
<version>${hdds.version}</version>
</dependency>
<dependency>
- <groupId>com.sun.xml.bind</groupId>
- <artifactId>jaxb-impl</artifactId>
- <version>2.3.0.1</version>
- </dependency>
- <dependency>
- <groupId>com.sun.xml.bind</groupId>
- <artifactId>jaxb-core</artifactId>
- <version>2.3.0.1</version>
- </dependency>
- <dependency>
- <groupId>javax.xml.bind</groupId>
- <artifactId>jaxb-api</artifactId>
- <version>2.3.0</version>
- </dependency>
- <dependency>
- <groupId>javax.activation</groupId>
- <artifactId>activation</artifactId>
- <version>1.1.1</version>
- </dependency>
- <dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15on</artifactId>
<version>${bouncycastle.version}</version>
</dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- <version>2.6</version>
- </dependency>
</dependencies>
</dependencyManagement>
<dependencies>
@@ -281,6 +280,7 @@
<exclude>**/yarn.lock</exclude>
<exclude>**/ozone-recon-web/build/**</exclude>
<exclude>src/main/license/**</exclude>
+ <exclude>src/main/proto/proto.lock</exclude>
</excludes>
</configuration>
</plugin>
diff --git a/hadoop-ozone/recon-codegen/pom.xml b/hadoop-ozone/recon-codegen/pom.xml
index 5abf9b0..9b8780b 100644
--- a/hadoop-ozone/recon-codegen/pom.xml
+++ b/hadoop-ozone/recon-codegen/pom.xml
@@ -23,28 +23,24 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-ozone-reconcodegen</artifactId>
<name>Apache Hadoop Ozone Recon CodeGen</name>
- <properties>
- <jooq.version>3.11.10</jooq.version>
- </properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-common</artifactId>
</dependency>
<dependency>
- <groupId>org.xerial</groupId>
- <artifactId>sqlite-jdbc</artifactId>
- <version>3.25.2</version>
+ <groupId>org.apache.derby</groupId>
+ <artifactId>derby</artifactId>
+ <version>10.14.2.0</version>
</dependency>
<dependency>
<groupId>com.google.inject.extensions</groupId>
<artifactId>guice-multibindings</artifactId>
- <version>${guice.version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-jdbc</artifactId>
- <version>5.1.3.RELEASE</version>
+ <version>${spring.version}</version>
</dependency>
<dependency>
<groupId>org.jooq</groupId>
@@ -64,7 +60,6 @@
<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
- <version>${guice.version}</version>
</dependency>
</dependencies>
</project>
diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/JooqCodeGenerator.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/JooqCodeGenerator.java
index fce4e0b..246f039 100644
--- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/JooqCodeGenerator.java
+++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/JooqCodeGenerator.java
@@ -17,13 +17,19 @@
*/
package org.hadoop.ozone.recon.codegen;
+import static org.hadoop.ozone.recon.codegen.SqlDbUtils.DERBY_DRIVER_CLASS;
+import static org.hadoop.ozone.recon.codegen.SqlDbUtils.createNewDerbyDatabase;
+
import java.io.File;
+import java.nio.file.Paths;
import java.sql.SQLException;
import java.util.Set;
import javax.sql.DataSource;
import org.apache.commons.io.FileUtils;
+import org.apache.derby.jdbc.EmbeddedDataSource;
+import org.apache.hadoop.util.Time;
import org.hadoop.ozone.recon.schema.ReconSchemaDefinition;
import org.jooq.codegen.GenerationTool;
import org.jooq.meta.jaxb.Configuration;
@@ -31,11 +37,11 @@
import org.jooq.meta.jaxb.Generate;
import org.jooq.meta.jaxb.Generator;
import org.jooq.meta.jaxb.Jdbc;
+import org.jooq.meta.jaxb.Logging;
import org.jooq.meta.jaxb.Strategy;
import org.jooq.meta.jaxb.Target;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.sqlite.SQLiteDataSource;
import com.google.inject.AbstractModule;
import com.google.inject.Guice;
@@ -55,10 +61,11 @@
private static final Logger LOG =
LoggerFactory.getLogger(JooqCodeGenerator.class);
- private static final String SQLITE_DB =
- System.getProperty("java.io.tmpdir") + "/recon-generated-schema";
- private static final String JDBC_URL = "jdbc:sqlite:" + SQLITE_DB;
-
+ private static final String DB = Paths.get(
+ System.getProperty("java.io.tmpdir"),
+ "recon-generated-schema-" + Time.monotonicNow()).toString();
+ public static final String RECON_SCHEMA_NAME = "RECON";
+ private static final String JDBC_URL = "jdbc:derby:" + DB;
private final Set<ReconSchemaDefinition> allDefinitions;
@Inject
@@ -82,26 +89,25 @@
Configuration configuration =
new Configuration()
.withJdbc(new Jdbc()
- .withDriver("org.sqlite.JDBC")
- .withUrl(JDBC_URL)
- .withUser("sa")
- .withPassword("sa"))
+ .withDriver(DERBY_DRIVER_CLASS)
+ .withUrl(JDBC_URL))
.withGenerator(new Generator()
.withDatabase(new Database()
- .withName("org.jooq.meta.sqlite.SQLiteDatabase")
+ .withName("org.jooq.meta.derby.DerbyDatabase")
.withOutputSchemaToDefault(true)
.withIncludeTables(true)
- .withIncludePrimaryKeys(true))
+ .withIncludePrimaryKeys(true)
+ .withInputSchema(RECON_SCHEMA_NAME))
.withGenerate(new Generate()
.withDaos(true)
- .withEmptyCatalogs(true)
- .withEmptySchemas(true))
+ .withEmptyCatalogs(true))
.withStrategy(new Strategy().withName(
"org.hadoop.ozone.recon.codegen.TableNamingStrategy"))
.withTarget(new Target()
.withPackageName("org.hadoop.ozone.recon.schema")
.withClean(true)
- .withDirectory(outputDir)));
+ .withDirectory(outputDir)))
+ .withLogging(Logging.WARN);
GenerationTool.generate(configuration);
}
@@ -109,27 +115,32 @@
* Provider for embedded datasource.
*/
static class LocalDataSourceProvider implements Provider<DataSource> {
- private static SQLiteDataSource db;
-
+ private static EmbeddedDataSource dataSource;
static {
- db = new SQLiteDataSource();
- db.setUrl(JDBC_URL);
+ try {
+ createNewDerbyDatabase(JDBC_URL, RECON_SCHEMA_NAME);
+ } catch (Exception e) {
+ LOG.error("Error creating Recon Derby DB.", e);
+ }
+ dataSource = new EmbeddedDataSource();
+ dataSource.setDatabaseName(DB);
+ dataSource.setUser(RECON_SCHEMA_NAME);
}
@Override
public DataSource get() {
- return db;
+ return dataSource;
}
static void cleanup() {
- FileUtils.deleteQuietly(new File(SQLITE_DB));
+ FileUtils.deleteQuietly(new File(DB));
}
}
public static void main(String[] args) {
if (args.length < 1) {
throw new IllegalArgumentException("Missing required arguments: " +
- "Need a ouput directory for generated code.\nUsage: " +
+ "Need an output directory for generated code.\nUsage: " +
"org.apache.hadoop.ozone.recon.persistence.JooqCodeGenerator " +
"<outputDirectory>.");
}
diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSchemaGenerationModule.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSchemaGenerationModule.java
index 2de6da8..8272c2b 100644
--- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSchemaGenerationModule.java
+++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSchemaGenerationModule.java
@@ -17,6 +17,7 @@
*/
package org.hadoop.ozone.recon.codegen;
+import org.hadoop.ozone.recon.schema.ContainerSchemaDefinition;
import org.hadoop.ozone.recon.schema.ReconTaskSchemaDefinition;
import org.hadoop.ozone.recon.schema.ReconSchemaDefinition;
import org.hadoop.ozone.recon.schema.StatsSchemaDefinition;
@@ -36,6 +37,7 @@
Multibinder<ReconSchemaDefinition> schemaBinder =
Multibinder.newSetBinder(binder(), ReconSchemaDefinition.class);
schemaBinder.addBinding().to(UtilizationSchemaDefinition.class);
+ schemaBinder.addBinding().to(ContainerSchemaDefinition.class);
schemaBinder.addBinding().to(ReconTaskSchemaDefinition.class);
schemaBinder.addBinding().to(StatsSchemaDefinition.class);
}
diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSqlDbConfig.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSqlDbConfig.java
new file mode 100644
index 0000000..704d26b
--- /dev/null
+++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSqlDbConfig.java
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.hadoop.ozone.recon.codegen;
+
+import static java.util.concurrent.TimeUnit.SECONDS;
+
+import org.apache.hadoop.hdds.conf.Config;
+import org.apache.hadoop.hdds.conf.ConfigGroup;
+import org.apache.hadoop.hdds.conf.ConfigTag;
+import org.apache.hadoop.hdds.conf.ConfigType;
+
+/**
+ * The configuration class for the Recon SQL DB.
+ */
+@ConfigGroup(prefix = "ozone.recon.sql.db")
+public class ReconSqlDbConfig {
+
+ @Config(key = "driver",
+ type = ConfigType.STRING,
+ defaultValue = "org.apache.derby.jdbc.EmbeddedDriver",
+ tags = {ConfigTag.STORAGE, ConfigTag.RECON, ConfigTag.OZONE},
+ description = "Recon SQL DB driver class. Defaults to Derby."
+ )
+ private String driverClass;
+
+ public String getDriverClass() {
+ return driverClass;
+ }
+
+ public void setDriverClass(String driverClass) {
+ this.driverClass = driverClass;
+ }
+
+ @Config(key = "jdbc.url",
+ type = ConfigType.STRING,
+ defaultValue = "jdbc:derby:${ozone.recon.db.dir}/ozone_recon_derby.db",
+ tags = {ConfigTag.STORAGE, ConfigTag.RECON, ConfigTag.OZONE},
+ description = "Ozone Recon SQL database jdbc url."
+ )
+ private String jdbcUrl;
+
+ public String getJdbcUrl() {
+ return jdbcUrl;
+ }
+
+ public void setJdbcUrl(String jdbcUrl) {
+ this.jdbcUrl = jdbcUrl;
+ }
+
+ @Config(key = "username",
+ type = ConfigType.STRING,
+ defaultValue = "",
+ tags = {ConfigTag.STORAGE, ConfigTag.RECON, ConfigTag.OZONE},
+ description = "Ozone Recon SQL database username."
+ )
+ private String username;
+
+ public String getUsername() {
+ return username;
+ }
+
+ public void setUsername(String username) {
+ this.username = username;
+ }
+
+ @Config(key = "password",
+ type = ConfigType.STRING,
+ defaultValue = "",
+ tags = {ConfigTag.STORAGE, ConfigTag.RECON, ConfigTag.OZONE},
+ description = "Ozone Recon SQL database password."
+ )
+ private String password;
+
+ public String getPassword() {
+ return password;
+ }
+
+ public void setPassword(String password) {
+ this.password = password;
+ }
+
+ @Config(key = "auto.commit",
+ type = ConfigType.BOOLEAN,
+ defaultValue = "false",
+ tags = {ConfigTag.STORAGE, ConfigTag.RECON, ConfigTag.OZONE},
+ description = "Sets the Ozone Recon database connection property of " +
+ "auto-commit to true/false."
+ )
+ private boolean autoCommit;
+
+ public boolean isAutoCommit() {
+ return autoCommit;
+ }
+
+ public void setAutoCommit(boolean autoCommit) {
+ this.autoCommit = autoCommit;
+ }
+
+ @Config(key = "conn.timeout",
+ type = ConfigType.TIME,
+ defaultValue = "30000ms",
+ tags = {ConfigTag.STORAGE, ConfigTag.RECON, ConfigTag.OZONE},
+ description = "Sets time in milliseconds before call to getConnection " +
+ "is timed out."
+ )
+ private long connectionTimeout;
+
+ public long getConnectionTimeout() {
+ return connectionTimeout;
+ }
+
+ public void setConnectionTimeout(long connectionTimeout) {
+ this.connectionTimeout = connectionTimeout;
+ }
+
+ @Config(key = "conn.max.active",
+ type = ConfigType.INT,
+ defaultValue = "5",
+ tags = {ConfigTag.STORAGE, ConfigTag.RECON, ConfigTag.OZONE},
+ description = "The max active connections to the SQL database."
+ )
+ private int maxActiveConnections;
+
+ public int getMaxActiveConnections() {
+ return maxActiveConnections;
+ }
+
+ public void setMaxActiveConnections(int maxActiveConnections) {
+ this.maxActiveConnections = maxActiveConnections;
+ }
+
+ @Config(key = "conn.max.age",
+ type = ConfigType.TIME, timeUnit = SECONDS,
+ defaultValue = "1800s",
+ tags = {ConfigTag.STORAGE, ConfigTag.RECON, ConfigTag.OZONE},
+ description = "Sets maximum time a connection can be active in seconds."
+ )
+ private long connectionMaxAge;
+
+ public long getConnectionMaxAge() {
+ return connectionMaxAge;
+ }
+
+ public void setConnectionMaxAge(long connectionMaxAge) {
+ this.connectionMaxAge = connectionMaxAge;
+ }
+
+ @Config(key = "conn.idle.max.age",
+ type = ConfigType.TIME, timeUnit = SECONDS,
+ defaultValue = "3600s",
+ tags = {ConfigTag.STORAGE, ConfigTag.RECON, ConfigTag.OZONE},
+ description = "Sets maximum time to live for idle connection in seconds."
+ )
+ private long connectionIdleMaxAge;
+
+ public long getConnectionIdleMaxAge() {
+ return connectionIdleMaxAge;
+ }
+
+ public void setConnectionIdleMaxAge(long connectionIdleMaxAge) {
+ this.connectionIdleMaxAge = connectionIdleMaxAge;
+ }
+
+ @Config(key = "conn.idle.test.period",
+ type = ConfigType.TIME, timeUnit = SECONDS,
+ defaultValue = "60s",
+ tags = {ConfigTag.STORAGE, ConfigTag.RECON, ConfigTag.OZONE},
+ description = "Sets maximum time to live for idle connection in seconds."
+ )
+ private long connectionIdleTestPeriod;
+
+ public long getConnectionIdleTestPeriod() {
+ return connectionIdleTestPeriod;
+ }
+
+ public void setConnectionIdleTestPeriod(long connectionIdleTestPeriod) {
+ this.connectionIdleTestPeriod = connectionIdleTestPeriod;
+ }
+
+ @Config(key = "conn.idle.test",
+ type = ConfigType.STRING,
+ defaultValue = "SELECT 1",
+ tags = {ConfigTag.STORAGE, ConfigTag.RECON, ConfigTag.OZONE},
+ description = "The query to send to the DB to maintain keep-alives and " +
+ "test for dead connections."
+ )
+ private String idleTestQuery;
+
+ public String getIdleTestQuery() {
+ return idleTestQuery;
+ }
+
+ public void setIdleTestQuery(String idleTestQuery) {
+ this.idleTestQuery = idleTestQuery;
+ }
+
+ @Config(key = "jooq.dialect",
+ type = ConfigType.STRING,
+ defaultValue = "DERBY",
+ tags = {ConfigTag.STORAGE, ConfigTag.RECON, ConfigTag.OZONE},
+ description = "Recon internally uses Jooq to talk to its SQL DB. By " +
+ "default, we support Derby and Sqlite out of the box. Please refer " +
+ "to https://www.jooq.org/javadoc/latest/org" +
+ ".jooq/org/jooq/SQLDialect.html to specify different dialect."
+ )
+ private String sqlDbDialect;
+
+ public String getSqlDbDialect() {
+ return sqlDbDialect;
+ }
+
+ public void setSqlDbDialect(String sqlDbDialect) {
+ this.sqlDbDialect = sqlDbDialect;
+ }
+
+ /**
+ * Class to hold config keys related to Recon SQL DB.
+ */
+ public static class ConfigKeys {
+ public static final String OZONE_RECON_SQL_DB_JDBC_URL =
+ "ozone.recon.sql.db.jdbc.url";
+ }
+}
diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/SqlDbUtils.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/SqlDbUtils.java
new file mode 100644
index 0000000..7e68541
--- /dev/null
+++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/SqlDbUtils.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.hadoop.ozone.recon.codegen;
+
+import static org.jooq.impl.DSL.count;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.function.BiPredicate;
+
+import org.jooq.exception.DataAccessException;
+import org.jooq.impl.DSL;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Constants and Helper functions for Recon SQL related stuff.
+ */
+public final class SqlDbUtils {
+
+ public final static String DERBY_DRIVER_CLASS =
+ "org.apache.derby.jdbc.EmbeddedDriver";
+ public final static String SQLITE_DRIVER_CLASS = "org.sqlite.JDBC";
+ public final static String DERBY_DISABLE_LOG_METHOD =
+ SqlDbUtils.class.getName() + ".disableDerbyLogFile";
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SqlDbUtils.class);
+
+ private SqlDbUtils() {
+ }
+
+ /**
+ * Create new Derby Database with URL and schema name.
+ * @param jdbcUrl JDBC url.
+ * @param schemaName Schema name
+ * @throws ClassNotFoundException on not finding driver class.
+ * @throws SQLException on SQL exception.
+ */
+ public static void createNewDerbyDatabase(String jdbcUrl, String schemaName)
+ throws ClassNotFoundException, SQLException {
+ System.setProperty("derby.stream.error.method",
+ DERBY_DISABLE_LOG_METHOD);
+ Class.forName(DERBY_DRIVER_CLASS);
+ try(Connection connection = DriverManager.getConnection(jdbcUrl
+ + ";user=" + schemaName
+ + ";create=true")) {
+ LOG.info("Created derby database at {}.", jdbcUrl);
+ }
+ }
+
+ /**
+ * Used to suppress embedded derby database logging.
+ * @return No-Op output stream.
+ */
+ public static OutputStream disableDerbyLogFile(){
+ return new OutputStream() {
+ public void write(int b) throws IOException {
+ // Ignore all log messages
+ }
+ };
+ }
+
+ /**
+ * Helper function to check if table exists through JOOQ.
+ */
+ public static final BiPredicate<Connection, String> TABLE_EXISTS_CHECK =
+ (conn, tableName) -> {
+ try {
+ DSL.using(conn).select(count()).from(tableName).execute();
+ } catch (DataAccessException ex) {
+ LOG.debug(ex.getMessage());
+ return false;
+ }
+ LOG.info("{} table already exists, skipping creation.", tableName);
+ return true;
+ };
+}
diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ContainerSchemaDefinition.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ContainerSchemaDefinition.java
new file mode 100644
index 0000000..ed60094
--- /dev/null
+++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ContainerSchemaDefinition.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.hadoop.ozone.recon.schema;
+
+import static org.hadoop.ozone.recon.codegen.SqlDbUtils.TABLE_EXISTS_CHECK;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import org.jooq.DSLContext;
+import org.jooq.impl.DSL;
+import org.jooq.impl.SQLDataType;
+
+import javax.sql.DataSource;
+import java.sql.Connection;
+import java.sql.SQLException;
+
+/**
+ * Class used to create tables that are required for tracking containers.
+ */
+@Singleton
+public class ContainerSchemaDefinition implements ReconSchemaDefinition {
+
+ public static final String CONTAINER_HISTORY_TABLE_NAME =
+ "CONTAINER_HISTORY";
+ public static final String MISSING_CONTAINERS_TABLE_NAME =
+ "MISSING_CONTAINERS";
+ private static final String CONTAINER_ID = "container_id";
+ private final DataSource dataSource;
+ private DSLContext dslContext;
+
+ @Inject
+ ContainerSchemaDefinition(DataSource dataSource) {
+ this.dataSource = dataSource;
+ }
+
+ @Override
+ public void initializeSchema() throws SQLException {
+ Connection conn = dataSource.getConnection();
+ dslContext = DSL.using(conn);
+ if (!TABLE_EXISTS_CHECK.test(conn, CONTAINER_HISTORY_TABLE_NAME)) {
+ createContainerHistoryTable();
+ }
+ if (!TABLE_EXISTS_CHECK.test(conn, MISSING_CONTAINERS_TABLE_NAME)) {
+ createMissingContainersTable();
+ }
+ }
+
+ /**
+ * Create the Container History table.
+ */
+ private void createContainerHistoryTable() {
+ dslContext.createTableIfNotExists(CONTAINER_HISTORY_TABLE_NAME)
+ .column(CONTAINER_ID, SQLDataType.BIGINT)
+ .column("datanode_host", SQLDataType.VARCHAR(1024))
+ .column("first_report_timestamp", SQLDataType.BIGINT)
+ .column("last_report_timestamp", SQLDataType.BIGINT)
+ .constraint(DSL.constraint("pk_container_id_datanode_host")
+ .primaryKey(CONTAINER_ID, "datanode_host"))
+ .execute();
+ }
+
+ /**
+ * Create the Missing Containers table.
+ */
+ private void createMissingContainersTable() {
+ dslContext.createTableIfNotExists(MISSING_CONTAINERS_TABLE_NAME)
+ .column(CONTAINER_ID, SQLDataType.BIGINT)
+ .column("missing_since", SQLDataType.BIGINT)
+ .constraint(DSL.constraint("pk_container_id")
+ .primaryKey(CONTAINER_ID))
+ .execute();
+ }
+
+ public DSLContext getDSLContext() {
+ return dslContext;
+ }
+}
diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ReconTaskSchemaDefinition.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ReconTaskSchemaDefinition.java
index 1856cc2..45fc1ba 100644
--- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ReconTaskSchemaDefinition.java
+++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/ReconTaskSchemaDefinition.java
@@ -18,11 +18,14 @@
package org.hadoop.ozone.recon.schema;
+import static org.hadoop.ozone.recon.codegen.SqlDbUtils.TABLE_EXISTS_CHECK;
+
import java.sql.Connection;
import java.sql.SQLException;
import javax.sql.DataSource;
+import com.google.inject.Singleton;
import org.jooq.impl.DSL;
import org.jooq.impl.SQLDataType;
@@ -32,10 +35,11 @@
* Class used to create tables that are required for Recon's task
* management.
*/
+@Singleton
public class ReconTaskSchemaDefinition implements ReconSchemaDefinition {
public static final String RECON_TASK_STATUS_TABLE_NAME =
- "recon_task_status";
+ "RECON_TASK_STATUS";
private final DataSource dataSource;
@Inject
@@ -46,14 +50,16 @@
@Override
public void initializeSchema() throws SQLException {
Connection conn = dataSource.getConnection();
- createReconTaskStatus(conn);
+ if (!TABLE_EXISTS_CHECK.test(conn, RECON_TASK_STATUS_TABLE_NAME)) {
+ createReconTaskStatusTable(conn);
+ }
}
/**
* Create the Recon Task Status table.
* @param conn connection
*/
- private void createReconTaskStatus(Connection conn) {
+ private void createReconTaskStatusTable(Connection conn) {
DSL.using(conn).createTableIfNotExists(RECON_TASK_STATUS_TABLE_NAME)
.column("task_name", SQLDataType.VARCHAR(1024))
.column("last_updated_timestamp", SQLDataType.BIGINT)
diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/StatsSchemaDefinition.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/StatsSchemaDefinition.java
index 6763bc8..adfaca6 100644
--- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/StatsSchemaDefinition.java
+++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/StatsSchemaDefinition.java
@@ -18,7 +18,10 @@
package org.hadoop.ozone.recon.schema;
+import static org.hadoop.ozone.recon.codegen.SqlDbUtils.TABLE_EXISTS_CHECK;
+
import com.google.inject.Inject;
+import com.google.inject.Singleton;
import org.jooq.impl.DSL;
import org.jooq.impl.SQLDataType;
@@ -29,9 +32,10 @@
/**
* Class used to create tables that are required for storing Ozone statistics.
*/
+@Singleton
public class StatsSchemaDefinition implements ReconSchemaDefinition {
- public static final String GLOBAL_STATS_TABLE_NAME = "global_stats";
+ public static final String GLOBAL_STATS_TABLE_NAME = "GLOBAL_STATS";
private final DataSource dataSource;
@Inject
@@ -42,7 +46,9 @@
@Override
public void initializeSchema() throws SQLException {
Connection conn = dataSource.getConnection();
- createGlobalStatsTable(conn);
+ if (!TABLE_EXISTS_CHECK.test(conn, GLOBAL_STATS_TABLE_NAME)) {
+ createGlobalStatsTable(conn);
+ }
}
/**
diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/UtilizationSchemaDefinition.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/UtilizationSchemaDefinition.java
index bc48c38..941a3c6 100644
--- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/UtilizationSchemaDefinition.java
+++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/UtilizationSchemaDefinition.java
@@ -17,13 +17,19 @@
*/
package org.hadoop.ozone.recon.schema;
+import static org.hadoop.ozone.recon.codegen.SqlDbUtils.TABLE_EXISTS_CHECK;
+
import java.sql.Connection;
import java.sql.SQLException;
import javax.sql.DataSource;
+import com.google.inject.Singleton;
+
import org.jooq.impl.DSL;
import org.jooq.impl.SQLDataType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.springframework.transaction.annotation.Transactional;
import com.google.inject.Inject;
@@ -31,18 +37,18 @@
/**
* Programmatic definition of Recon DDL.
*/
+@Singleton
public class UtilizationSchemaDefinition implements ReconSchemaDefinition {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(UtilizationSchemaDefinition.class);
+
private final DataSource dataSource;
public static final String CLUSTER_GROWTH_DAILY_TABLE_NAME =
- "cluster_growth_daily";
-
+ "CLUSTER_GROWTH_DAILY";
public static final String FILE_COUNT_BY_SIZE_TABLE_NAME =
- "file_count_by_size";
-
- public static final String MISSING_CONTAINERS_TABLE_NAME =
- "missing_containers";
+ "FILE_COUNT_BY_SIZE";
@Inject
UtilizationSchemaDefinition(DataSource dataSource) {
@@ -53,12 +59,15 @@
@Transactional
public void initializeSchema() throws SQLException {
Connection conn = dataSource.getConnection();
- createClusterGrowthTable(conn);
- createFileSizeCount(conn);
- createMissingContainersTable(conn);
+ if (!TABLE_EXISTS_CHECK.test(conn, FILE_COUNT_BY_SIZE_TABLE_NAME)) {
+ createFileSizeCountTable(conn);
+ }
+ if (!TABLE_EXISTS_CHECK.test(conn, CLUSTER_GROWTH_DAILY_TABLE_NAME)) {
+ createClusterGrowthTable(conn);
+ }
}
- void createClusterGrowthTable(Connection conn) {
+ private void createClusterGrowthTable(Connection conn) {
DSL.using(conn).createTableIfNotExists(CLUSTER_GROWTH_DAILY_TABLE_NAME)
.column("timestamp", SQLDataType.TIMESTAMP)
.column("datanode_id", SQLDataType.INTEGER)
@@ -73,7 +82,7 @@
.execute();
}
- void createFileSizeCount(Connection conn) {
+ private void createFileSizeCountTable(Connection conn) {
DSL.using(conn).createTableIfNotExists(FILE_COUNT_BY_SIZE_TABLE_NAME)
.column("file_size", SQLDataType.BIGINT)
.column("count", SQLDataType.BIGINT)
@@ -81,13 +90,4 @@
.primaryKey("file_size"))
.execute();
}
-
- void createMissingContainersTable(Connection conn) {
- DSL.using(conn).createTableIfNotExists(MISSING_CONTAINERS_TABLE_NAME)
- .column("container_id", SQLDataType.BIGINT)
- .column("missing_since", SQLDataType.BIGINT)
- .constraint(DSL.constraint("pk_container_id")
- .primaryKey("container_id"))
- .execute();
- }
}
diff --git a/hadoop-ozone/recon/pom.xml b/hadoop-ozone/recon/pom.xml
index 96f9739..7d294bb 100644
--- a/hadoop-ozone/recon/pom.xml
+++ b/hadoop-ozone/recon/pom.xml
@@ -16,17 +16,13 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
- <artifactId>hadoop-ozone</artifactId>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone</artifactId>
<version>0.6.0-SNAPSHOT</version>
</parent>
<name>Apache Hadoop Ozone Recon</name>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-ozone-recon</artifactId>
- <properties>
- <jooq.version>3.11.10</jooq.version>
- <spring.version>5.1.3.RELEASE</spring.version>
- </properties>
<build>
<resources>
<resource>
@@ -40,7 +36,6 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
- <version>${exec-maven-plugin.version}</version>
<executions>
<execution>
<phase>generate-resources</phase>
@@ -91,7 +86,6 @@
<plugin>
<groupId>com.github.eirslett</groupId>
<artifactId>frontend-maven-plugin</artifactId>
- <version>1.6</version>
<configuration>
<installDirectory>target</installDirectory>
<workingDirectory>${basedir}/src/main/resources/webapps/recon/ozone-recon-web</workingDirectory>
@@ -103,8 +97,8 @@
<goal>install-node-and-yarn</goal>
</goals>
<configuration>
- <nodeVersion>v12.1.0</nodeVersion>
- <yarnVersion>v1.9.2</yarnVersion>
+ <nodeVersion>v12.14.1</nodeVersion>
+ <yarnVersion>v1.22.4</yarnVersion>
</configuration>
</execution>
<execution>
@@ -194,6 +188,28 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-ozone-manager</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-test-utils</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <!-- visible only for ContainerOperationClient -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-tools</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -202,68 +218,38 @@
<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
- <version>${guice.version}</version>
- </dependency>
- <dependency>
- <groupId>com.google.inject.extensions</groupId>
- <artifactId>guice-servlet</artifactId>
- <version>${guice.version}</version>
- </dependency>
- <dependency>
- <groupId>org.glassfish.jersey.containers</groupId>
- <artifactId>jersey-container-servlet</artifactId>
- <version>2.27</version>
- <exclusions>
- <exclusion>
- <groupId>org.glassfish.hk2</groupId>
- <artifactId>hk2-api</artifactId>
- </exclusion>
- </exclusions>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.glassfish.jersey.containers</groupId>
- <artifactId>jersey-container-servlet-core</artifactId>
- <version>2.27</version>
- </dependency>
- <dependency>
- <groupId>org.glassfish.hk2</groupId>
- <artifactId>guice-bridge</artifactId>
- <version>2.5.0</version>
- </dependency>
- <dependency>
- <groupId>org.glassfish.jersey.core</groupId>
- <artifactId>jersey-server</artifactId>
- <version>2.27</version>
- </dependency>
- <dependency>
- <groupId>org.glassfish.jersey.media</groupId>
- <artifactId>jersey-media-json-jackson</artifactId>
- <version>2.27</version>
</dependency>
<dependency>
<groupId>com.google.inject.extensions</groupId>
<artifactId>guice-assistedinject</artifactId>
- <version>${guice.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.inject.extensions</groupId>
+ <artifactId>guice-servlet</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.containers</groupId>
+ <artifactId>jersey-container-servlet</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.containers</groupId>
+ <artifactId>jersey-container-servlet-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.hk2</groupId>
+ <artifactId>guice-bridge</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.core</groupId>
+ <artifactId>jersey-server</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.media</groupId>
+ <artifactId>jersey-media-json-jackson</artifactId>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.inject</groupId>
<artifactId>jersey-hk2</artifactId>
- <version>2.27</version>
- <exclusions>
- <exclusion>
- <artifactId>hk2-api</artifactId>
- <groupId>org.glassfish.hk2</groupId>
- </exclusion>
- <exclusion>
- <groupId>org.glassfish.hk2.external</groupId>
- <artifactId>aopalliance-repackaged</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.glassfish.hk2</groupId>
- <artifactId>hk2-utils</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>junit</groupId>
@@ -277,9 +263,8 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
+ <artifactId>hadoop-hdds-hadoop-dependency-test</artifactId>
<scope>test</scope>
- <type>test-jar</type>
<exclusions>
<exclusion>
<artifactId>jersey-server</artifactId>
@@ -317,12 +302,15 @@
<dependency>
<groupId>com.jolbox</groupId>
<artifactId>bonecp</artifactId>
- <version>0.8.0.RELEASE</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.derby</groupId>
+ <artifactId>derby</artifactId>
+ <version>10.14.2.0</version>
</dependency>
<dependency>
<groupId>org.xerial</groupId>
<artifactId>sqlite-jdbc</artifactId>
- <version>3.25.2</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
@@ -332,7 +320,6 @@
<dependency>
<groupId>javax.activation</groupId>
<artifactId>activation</artifactId>
- <version>1.1.1</version>
</dependency>
<dependency>
<groupId>com.github.spotbugs</groupId>
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ConfigurationProvider.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ConfigurationProvider.java
index 5a6a916..6312365 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ConfigurationProvider.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ConfigurationProvider.java
@@ -17,10 +17,12 @@
*/
package org.apache.hadoop.ozone.recon;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configuration.DeprecationDelta;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+
import com.google.common.annotations.VisibleForTesting;
import com.google.inject.Provider;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
/**
* Ozone Configuration Provider.
@@ -38,9 +40,15 @@
private static OzoneConfiguration configuration;
private static void addDeprecations() {
- Configuration.addDeprecation(
- ReconServerConfigKeys.OZONE_RECON_HTTP_KEYTAB_FILE_OLD,
- ReconServerConfigKeys.OZONE_RECON_HTTP_KEYTAB_FILE);
+ Configuration.addDeprecations(new DeprecationDelta[]{
+ new DeprecationDelta("ozone.recon.keytab.file",
+ ReconServerConfigKeys.OZONE_RECON_HTTP_KEYTAB_FILE),
+ new DeprecationDelta(("ozone.recon.http.kerberos.keytab.file"),
+ ReconServerConfigKeys.OZONE_RECON_HTTP_KEYTAB_FILE),
+ new DeprecationDelta("ozone.recon.authentication.kerberos.principal",
+ ReconServerConfigKeys.
+ OZONE_RECON_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL)
+ });
}
@VisibleForTesting
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java
index 8ceb1e0..590997e 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java
@@ -19,19 +19,10 @@
import static org.apache.hadoop.hdds.scm.cli.ContainerOperationClient.newContainerRpcClient;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_INTERNAL_SERVICE_ID;
-import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SQL_AUTO_COMMIT;
-import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SQL_CONNECTION_TIMEOUT;
-import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SQL_DB_DRIVER;
-import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SQL_DB_JDBC_URL;
-import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SQL_DB_PASSWORD;
-import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SQL_DB_USER;
-import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SQL_IDLE_CONNECTION_TEST_PERIOD;
-import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SQL_MAX_ACTIVE_CONNECTIONS;
-import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SQL_MAX_CONNECTION_AGE;
-import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SQL_MAX_IDLE_CONNECTION_AGE;
-import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SQL_MAX_IDLE_CONNECTION_TEST_STMT;
import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.util.List;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
@@ -39,6 +30,7 @@
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB;
+import org.apache.hadoop.ozone.recon.persistence.ContainerSchemaManager;
import org.apache.hadoop.ozone.recon.persistence.DataSourceConfiguration;
import org.apache.hadoop.ozone.recon.persistence.JooqPersistenceModule;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
@@ -59,13 +51,19 @@
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.hdds.utils.db.DBStore;
import org.apache.ratis.protocol.ClientId;
+import org.hadoop.ozone.recon.codegen.ReconSqlDbConfig;
+import org.hadoop.ozone.recon.schema.tables.daos.ClusterGrowthDailyDao;
+import org.hadoop.ozone.recon.schema.tables.daos.ContainerHistoryDao;
import org.hadoop.ozone.recon.schema.tables.daos.FileCountBySizeDao;
+import org.hadoop.ozone.recon.schema.tables.daos.GlobalStatsDao;
import org.hadoop.ozone.recon.schema.tables.daos.MissingContainersDao;
import org.hadoop.ozone.recon.schema.tables.daos.ReconTaskStatusDao;
import org.jooq.Configuration;
+import org.jooq.DAO;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.ImmutableList;
import com.google.inject.AbstractModule;
import com.google.inject.Provides;
import com.google.inject.Singleton;
@@ -88,6 +86,7 @@
.to(ReconOmMetadataManagerImpl.class);
bind(OMMetadataManager.class).to(ReconOmMetadataManagerImpl.class);
+ bind(ContainerSchemaManager.class).in(Singleton.class);
bind(ContainerDBServiceProvider.class)
.to(ContainerDBServiceProviderImpl.class).in(Singleton.class);
bind(OzoneManagerServiceProvider.class)
@@ -98,6 +97,7 @@
getProvider(DataSourceConfiguration.class)));
install(new ReconOmTaskBindingModule());
+ install(new ReconDaoBindingModule());
bind(ReconTaskController.class)
.to(ReconTaskControllerImpl.class).in(Singleton.class);
@@ -107,21 +107,6 @@
.to(ReconStorageContainerManagerFacade.class).in(Singleton.class);
}
- @Provides
- ReconTaskStatusDao getReconTaskTableDao(final Configuration sqlConfig) {
- return new ReconTaskStatusDao(sqlConfig);
- }
-
- @Provides
- MissingContainersDao getMissingContainersDao(final Configuration sqlConfig) {
- return new MissingContainersDao(sqlConfig);
- }
-
- @Provides
- FileCountBySizeDao getFileCountBySizeDao(final Configuration sqlConfig) {
- return new FileCountBySizeDao(sqlConfig);
- }
-
static class ReconOmTaskBindingModule extends AbstractModule {
@Override
protected void configure() {
@@ -132,6 +117,32 @@
}
}
+ /**
+ * Class that has all the DAO bindings in Recon.
+ */
+ public static class ReconDaoBindingModule extends AbstractModule {
+ public static final List<Class<? extends DAO>> RECON_DAO_LIST =
+ ImmutableList.of(
+ FileCountBySizeDao.class,
+ ReconTaskStatusDao.class,
+ MissingContainersDao.class,
+ GlobalStatsDao.class,
+ ClusterGrowthDailyDao.class,
+ ContainerHistoryDao.class);
+
+ @Override
+ protected void configure() {
+ RECON_DAO_LIST.forEach(aClass -> {
+ try {
+ bind(aClass).toConstructor(
+ (Constructor) aClass.getConstructor(Configuration.class));
+ } catch (NoSuchMethodException e) {
+ LOG.error("Error creating DAO {} ", aClass.getSimpleName(), e);
+ }
+ });
+ }
+ }
+
@Provides
OzoneManagerProtocol getOzoneManagerProtocol(
final OzoneConfiguration ozoneConfiguration) {
@@ -166,73 +177,68 @@
DataSourceConfiguration getDataSourceConfiguration(
final OzoneConfiguration ozoneConfiguration) {
+ ReconSqlDbConfig sqlDbConfig =
+ ozoneConfiguration.getObject(ReconSqlDbConfig.class);
+
return new DataSourceConfiguration() {
@Override
public String getDriverClass() {
- return ozoneConfiguration.get(OZONE_RECON_SQL_DB_DRIVER,
- "org.sqlite.JDBC");
+ return sqlDbConfig.getDriverClass();
}
@Override
public String getJdbcUrl() {
- return ozoneConfiguration.get(OZONE_RECON_SQL_DB_JDBC_URL);
+ return sqlDbConfig.getJdbcUrl();
}
@Override
public String getUserName() {
- return ozoneConfiguration.get(OZONE_RECON_SQL_DB_USER);
+ return sqlDbConfig.getUsername();
}
@Override
public String getPassword() {
- return ozoneConfiguration.get(OZONE_RECON_SQL_DB_PASSWORD);
+ return sqlDbConfig.getPassword();
}
@Override
public boolean setAutoCommit() {
- return ozoneConfiguration.getBoolean(
- OZONE_RECON_SQL_AUTO_COMMIT, false);
+ return sqlDbConfig.isAutoCommit();
}
@Override
public long getConnectionTimeout() {
- return ozoneConfiguration.getLong(
- OZONE_RECON_SQL_CONNECTION_TIMEOUT, 30000);
+ return sqlDbConfig.getConnectionTimeout();
}
@Override
public String getSqlDialect() {
- return JooqPersistenceModule.DEFAULT_DIALECT.toString();
+ return sqlDbConfig.getSqlDbDialect();
}
@Override
public Integer getMaxActiveConnections() {
- return ozoneConfiguration.getInt(
- OZONE_RECON_SQL_MAX_ACTIVE_CONNECTIONS, 10);
+ return sqlDbConfig.getMaxActiveConnections();
}
@Override
- public Integer getMaxConnectionAge() {
- return ozoneConfiguration.getInt(
- OZONE_RECON_SQL_MAX_CONNECTION_AGE, 1800);
+ public long getMaxConnectionAge() {
+ return sqlDbConfig.getConnectionMaxAge();
}
@Override
- public Integer getMaxIdleConnectionAge() {
- return ozoneConfiguration.getInt(
- OZONE_RECON_SQL_MAX_IDLE_CONNECTION_AGE, 3600);
+ public long getMaxIdleConnectionAge() {
+ return sqlDbConfig.getConnectionIdleMaxAge();
}
@Override
public String getConnectionTestStatement() {
- return ozoneConfiguration.get(
- OZONE_RECON_SQL_MAX_IDLE_CONNECTION_TEST_STMT, "SELECT 1");
+ return sqlDbConfig.getIdleTestQuery();
}
@Override
- public Integer getIdleConnectionTestPeriod() {
- return ozoneConfiguration.getInt(
- OZONE_RECON_SQL_IDLE_CONNECTION_TEST_PERIOD, 60);
+ public long getIdleConnectionTestPeriod() {
+ return sqlDbConfig.getConnectionIdleTestPeriod();
}
};
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconHttpServer.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconHttpServer.java
index c66a487..fbf5170 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconHttpServer.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconHttpServer.java
@@ -85,4 +85,14 @@
protected String getEnabledKey() {
return ReconServerConfigKeys.OZONE_RECON_HTTP_ENABLED_KEY;
}
+
+ @Override
+ protected String getHttpAuthType() {
+ return ReconServerConfigKeys.OZONE_RECON_HTTP_AUTH_TYPE;
+ }
+
+ @Override
+ protected String getHttpAuthConfigPrefix() {
+ return ReconServerConfigKeys.OZONE_RECON_HTTP_AUTH_CONFIG_PREFIX;
+ }
}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaManager.java
index 1ca0b94..883f90a 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaManager.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaManager.java
@@ -26,6 +26,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.annotations.VisibleForTesting;
import com.google.inject.Inject;
/**
@@ -42,7 +43,8 @@
this.reconSchemaDefinitions.addAll(reconSchemaDefinitions);
}
- void createReconSchema() {
+ @VisibleForTesting
+ public void createReconSchema() {
reconSchemaDefinitions.forEach(reconSchemaDefinition -> {
try {
reconSchemaDefinition.initializeSchema();
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java
index 07353ae..ee4d5a6 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java
@@ -26,6 +26,7 @@
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.recon.ReconConfig;
import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
+import org.apache.hadoop.hdds.utils.HddsServerUtil;
import org.apache.hadoop.ozone.OzoneSecurityUtil;
import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider;
import org.apache.hadoop.ozone.recon.spi.OzoneManagerServiceProvider;
@@ -56,6 +57,7 @@
private ContainerDBServiceProvider containerDBServiceProvider;
private OzoneManagerServiceProvider ozoneManagerServiceProvider;
private OzoneStorageContainerManager reconStorageContainerManager;
+ private OzoneConfiguration configuration;
private volatile boolean isStarted = false;
@@ -65,8 +67,8 @@
@Override
public Void call() throws Exception {
- OzoneConfiguration ozoneConfiguration = createOzoneConfiguration();
- ConfigurationProvider.setConfiguration(ozoneConfiguration);
+ configuration = createOzoneConfiguration();
+ ConfigurationProvider.setConfiguration(configuration);
injector = Guice.createInjector(new
ReconControllerModule(),
@@ -83,7 +85,7 @@
LOG.info("Initializing Recon server...");
try {
- loginReconUserIfSecurityEnabled(ozoneConfiguration);
+ loginReconUserIfSecurityEnabled(configuration);
this.containerDBServiceProvider =
injector.getInstance(ContainerDBServiceProvider.class);
@@ -124,6 +126,8 @@
if (!isStarted) {
LOG.info("Starting Recon server");
isStarted = true;
+ // Initialize metrics for Recon
+ HddsServerUtil.initializeMetrics(configuration, "Recon");
if (httpServer != null) {
httpServer.start();
}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServerConfigKeys.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServerConfigKeys.java
index 9d037c3..3605896 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServerConfigKeys.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServerConfigKeys.java
@@ -39,17 +39,14 @@
"ozone.recon.http-address";
public static final String OZONE_RECON_HTTPS_ADDRESS_KEY =
"ozone.recon.https-address";
- // Deprecated config
- public static final String OZONE_RECON_HTTP_KEYTAB_FILE_OLD =
- "ozone.recon.keytab.file";
public static final String OZONE_RECON_HTTP_KEYTAB_FILE =
- "ozone.recon.http.kerberos.keytab.file";
+ "ozone.recon.http.auth.kerberos.keytab";
public static final String OZONE_RECON_HTTP_BIND_HOST_DEFAULT =
"0.0.0.0";
public static final int OZONE_RECON_HTTP_BIND_PORT_DEFAULT = 9888;
public static final int OZONE_RECON_HTTPS_BIND_PORT_DEFAULT = 9889;
public static final String OZONE_RECON_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL =
- "ozone.recon.authentication.kerberos.principal";
+ "ozone.recon.http.auth.kerberos.principal";
public static final String OZONE_RECON_CONTAINER_DB_CACHE_SIZE_MB =
"ozone.recon.container.db.cache.size.mb";
@@ -96,34 +93,18 @@
public static final String RECON_OM_SNAPSHOT_TASK_FLUSH_PARAM =
"recon.om.snapshot.task.flush.param";
- // Persistence properties
- public static final String OZONE_RECON_SQL_DB_DRIVER =
- "ozone.recon.sql.db.driver";
- public static final String OZONE_RECON_SQL_DB_JDBC_URL =
- "ozone.recon.sql.db.jdbc.url";
- public static final String OZONE_RECON_SQL_DB_USER =
- "ozone.recon.sql.db.username";
- public static final String OZONE_RECON_SQL_DB_PASSWORD =
- "ozone.recon.sql.db.password";
- public static final String OZONE_RECON_SQL_AUTO_COMMIT =
- "ozone.recon.sql.db.auto.commit";
- public static final String OZONE_RECON_SQL_CONNECTION_TIMEOUT =
- "ozone.recon.sql.db.conn.timeout";
- public static final String OZONE_RECON_SQL_MAX_ACTIVE_CONNECTIONS =
- "ozone.recon.sql.db.conn.max.active";
- public static final String OZONE_RECON_SQL_MAX_CONNECTION_AGE =
- "ozone.recon.sql.db.conn.max.age";
- public static final String OZONE_RECON_SQL_MAX_IDLE_CONNECTION_AGE =
- "ozone.recon.sql.db.conn.idle.max.age";
- public static final String OZONE_RECON_SQL_IDLE_CONNECTION_TEST_PERIOD =
- "ozone.recon.sql.db.conn.idle.test.period";
- public static final String OZONE_RECON_SQL_MAX_IDLE_CONNECTION_TEST_STMT =
- "ozone.recon.sql.db.conn.idle.test";
-
public static final String OZONE_RECON_TASK_THREAD_COUNT_KEY =
"ozone.recon.task.thread.count";
public static final int OZONE_RECON_TASK_THREAD_COUNT_DEFAULT = 5;
+ public static final String OZONE_RECON_HTTP_AUTH_CONFIG_PREFIX =
+ "ozone.recon.http.auth.";
+
+ public static final String OZONE_RECON_HTTP_AUTH_TYPE =
+ OZONE_RECON_HTTP_AUTH_CONFIG_PREFIX + "type";
+
+
+
/**
* Private constructor for utility class.
*/
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java
index 9223456..cba7428 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java
@@ -18,12 +18,6 @@
package org.apache.hadoop.ozone.recon;
-import static java.net.HttpURLConnection.HTTP_CREATED;
-import static java.net.HttpURLConnection.HTTP_OK;
-import static org.apache.hadoop.hdds.server.ServerUtils.getDirectoryFromConfig;
-import static org.apache.hadoop.hdds.server.ServerUtils.getOzoneMetaDirPath;
-import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SCM_DB_DIR;
-
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
@@ -31,24 +25,27 @@
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.net.URL;
+import java.net.URLConnection;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.zip.GZIPOutputStream;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.HddsUtils;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdfs.web.URLConnectionFactory;
+import org.apache.hadoop.io.IOUtils;
+
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.HddsUtils;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.impl.client.CloseableHttpClient;
+import static org.apache.hadoop.hdds.server.ServerUtils.getDirectoryFromConfig;
+import static org.apache.hadoop.hdds.server.ServerUtils.getOzoneMetaDirPath;
+import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SCM_DB_DIR;
-import org.apache.http.util.EntityUtils;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -65,7 +62,7 @@
private static final Logger LOG = LoggerFactory.getLogger(
ReconUtils.class);
- public static File getReconScmDbDir(Configuration conf) {
+ public static File getReconScmDbDir(ConfigurationSource conf) {
return new ReconUtils().getReconDbDir(conf, OZONE_RECON_SCM_DB_DIR);
}
@@ -77,7 +74,7 @@
* @param dirConfigKey key to check
* @return Return File based on configured or fallback value.
*/
- public File getReconDbDir(Configuration conf, String dirConfigKey) {
+ public File getReconDbDir(ConfigurationSource conf, String dirConfigKey) {
File metadataDir = getDirectoryFromConfig(conf, dirConfigKey,
"Recon");
@@ -219,30 +216,19 @@
/**
* Make HTTP GET call on the URL and return inputstream to the response.
- * @param httpClient HttpClient to use.
+ * @param connectionFactory URLConnectionFactory to use.
* @param url url to call
+ * @param isSpnego is SPNEGO enabled
* @return Inputstream to the response of the HTTP call.
- * @throws IOException While reading the response.
+ * @throws IOException, AuthenticationException While reading the response.
*/
- public InputStream makeHttpCall(CloseableHttpClient httpClient, String url)
- throws IOException {
-
- HttpGet httpGet = new HttpGet(url);
- HttpResponse response = httpClient.execute(httpGet);
- int errorCode = response.getStatusLine().getStatusCode();
- HttpEntity entity = response.getEntity();
-
- if ((errorCode == HTTP_OK) || (errorCode == HTTP_CREATED)) {
- return entity.getContent();
- }
-
- if (entity != null) {
- throw new IOException("Unexpected exception when trying to reach Ozone " +
- "Manager, " + EntityUtils.toString(entity));
- } else {
- throw new IOException("Unexpected null in http payload," +
- " while processing request");
- }
+ public InputStream makeHttpCall(URLConnectionFactory connectionFactory,
+ String url, boolean isSpnego)
+ throws IOException, AuthenticationException {
+ URLConnection urlConnection =
+ connectionFactory.openConnection(new URL(url), isSpnego);
+ urlConnection.connect();
+ return urlConnection.getInputStream();
}
/**
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ClusterStateEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ClusterStateEndpoint.java
index e43dbf8..918ee18 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ClusterStateEndpoint.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ClusterStateEndpoint.java
@@ -80,23 +80,25 @@
new DatanodeStorageReport(stats.getCapacity().get(),
stats.getScmUsed().get(), stats.getRemaining().get());
ClusterStateResponse.Builder builder = ClusterStateResponse.newBuilder();
- try {
- builder.setVolumes(
- omMetadataManager.getVolumeTable().getEstimatedKeyCount());
- } catch (Exception ex) {
- LOG.error("Unable to get Volumes count in ClusterStateResponse.", ex);
- }
- try {
- builder.setBuckets(
- omMetadataManager.getBucketTable().getEstimatedKeyCount());
- } catch (Exception ex) {
- LOG.error("Unable to get Buckets count in ClusterStateResponse.", ex);
- }
- try {
- builder.setKeys(
- omMetadataManager.getKeyTable().getEstimatedKeyCount());
- } catch (Exception ex) {
- LOG.error("Unable to get Keys count in ClusterStateResponse.", ex);
+ if (omMetadataManager.isOmTablesInitialized()) {
+ try {
+ builder.setVolumes(
+ omMetadataManager.getVolumeTable().getEstimatedKeyCount());
+ } catch (Exception ex) {
+ LOG.error("Unable to get Volumes count in ClusterStateResponse.", ex);
+ }
+ try {
+ builder.setBuckets(
+ omMetadataManager.getBucketTable().getEstimatedKeyCount());
+ } catch (Exception ex) {
+ LOG.error("Unable to get Buckets count in ClusterStateResponse.", ex);
+ }
+ try {
+ builder.setKeys(
+ omMetadataManager.getKeyTable().getEstimatedKeyCount());
+ } catch (Exception ex) {
+ LOG.error("Unable to get Keys count in ClusterStateResponse.", ex);
+ }
}
ClusterStateResponse response = builder
.setStorageReport(storageReport)
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java
index b33db8d..325b0b9 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java
@@ -20,7 +20,6 @@
import java.io.IOException;
import java.time.Instant;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -53,9 +52,11 @@
import org.apache.hadoop.ozone.recon.api.types.KeysResponse;
import org.apache.hadoop.ozone.recon.api.types.MissingContainerMetadata;
import org.apache.hadoop.ozone.recon.api.types.MissingContainersResponse;
+import org.apache.hadoop.ozone.recon.persistence.ContainerSchemaManager;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
import org.apache.hadoop.ozone.recon.scm.ReconContainerManager;
import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider;
+import org.hadoop.ozone.recon.schema.tables.pojos.ContainerHistory;
import static org.apache.hadoop.ozone.recon.ReconConstants.DEFAULT_FETCH_COUNT;
import static org.apache.hadoop.ozone.recon.ReconConstants.PREV_CONTAINER_ID_DEFAULT_VALUE;
@@ -77,11 +78,14 @@
private ReconOMMetadataManager omMetadataManager;
private ReconContainerManager containerManager;
+ private ContainerSchemaManager containerSchemaManager;
@Inject
- public ContainerEndpoint(OzoneStorageContainerManager reconSCM) {
+ public ContainerEndpoint(OzoneStorageContainerManager reconSCM,
+ ContainerSchemaManager containerSchemaManager) {
this.containerManager =
(ReconContainerManager) reconSCM.getContainerManager();
+ this.containerSchemaManager = containerSchemaManager;
}
/**
@@ -204,6 +208,21 @@
}
/**
+ * Return Container replica history for the container identified by the id
+ * param.
+ *
+ * @param containerID the given containerID.
+ * @return {@link Response}
+ */
+ @GET
+ @Path("/{id}/replicaHistory")
+ public Response getReplicaHistoryForContainer(
+ @PathParam("id") Long containerID) {
+ return Response.ok(
+ containerSchemaManager.getAllContainerHistory(containerID)).build();
+ }
+
+ /**
* Return
* {@link org.apache.hadoop.ozone.recon.api.types.MissingContainerMetadata}
* for all missing containers.
@@ -222,9 +241,9 @@
long keyCount = containerInfo.getNumberOfKeys();
UUID pipelineID = containerInfo.getPipelineID().getId();
- // TODO: Find out which datanodes had replicas of this container
- // and populate this list
- List datanodes = Collections.emptyList();
+ List<ContainerHistory> datanodes =
+ containerSchemaManager.getLatestContainerHistory(
+ containerID, containerInfo.getReplicationFactor().getNumber());
missingContainers.add(new MissingContainerMetadata(containerID,
container.getMissingSince(), keyCount, pipelineID, datanodes));
} catch (IOException ioEx) {
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NodeEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NodeEndpoint.java
index d59bee6..2924435 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NodeEndpoint.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NodeEndpoint.java
@@ -38,9 +38,11 @@
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.ozone.recon.scm.ReconPipelineManager;
import org.slf4j.Logger;
@@ -81,31 +83,44 @@
String hostname = datanode.getHostName();
Set<PipelineID> pipelineIDs = nodeManager.getPipelines(datanode);
List<DatanodePipeline> pipelines = new ArrayList<>();
+ AtomicInteger leaderCount = new AtomicInteger();
+ DatanodeMetadata.Builder builder = DatanodeMetadata.newBuilder();
pipelineIDs.forEach(pipelineID -> {
try {
Pipeline pipeline = pipelineManager.getPipeline(pipelineID);
+ String leaderNode = pipeline.getLeaderNode().getHostName();
DatanodePipeline datanodePipeline = new DatanodePipeline(
pipelineID.getId(),
pipeline.getType().toString(),
- pipeline.getFactor().getNumber()
+ pipeline.getFactor().getNumber(),
+ leaderNode
);
pipelines.add(datanodePipeline);
+ if (pipeline.getLeaderId().equals(datanode.getUuid())) {
+ leaderCount.getAndIncrement();
+ }
} catch (PipelineNotFoundException ex) {
LOG.warn("Cannot get pipeline {} for datanode {}, pipeline not found",
pipelineID.getId(), hostname, ex);
+ } catch (IOException ioEx) {
+ LOG.warn("Cannot get leader node of pipeline with id {}.",
+ pipelineID.getId(), ioEx);
}
});
- int containers;
try {
- containers = nodeManager.getContainers(datanode).size();
+ int containers = nodeManager.getContainers(datanode).size();
+ builder.withContainers(containers);
} catch (NodeNotFoundException ex) {
- containers = 0;
LOG.warn("Cannot get containers, datanode {} not found.",
datanode.getUuid(), ex);
}
- long heartbeat = nodeManager.getLastHeartbeat(datanode);
- datanodes.add(new DatanodeMetadata(hostname, nodeState, heartbeat,
- storageReport, pipelines, containers));
+ datanodes.add(builder.withHostname(hostname)
+ .withDatanodeStorageReport(storageReport)
+ .withLastHeartbeat(nodeManager.getLastHeartbeat(datanode))
+ .withState(nodeState)
+ .withPipelines(pipelines)
+ .withLeaderCount(leaderCount.get())
+ .build());
});
DatanodesResponse datanodesResponse =
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DatanodeMetadata.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DatanodeMetadata.java
index 2378e8b..44490f1 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DatanodeMetadata.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DatanodeMetadata.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.ozone.recon.api.types;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
import javax.xml.bind.annotation.XmlAccessType;
@@ -28,7 +29,7 @@
* Metadata object that represents a Datanode.
*/
@XmlAccessorType(XmlAccessType.FIELD)
-public class DatanodeMetadata {
+public final class DatanodeMetadata {
@XmlElement(name = "hostname")
private String hostname;
@@ -48,18 +49,17 @@
@XmlElement(name = "containers")
private int containers;
- public DatanodeMetadata(String hostname,
- NodeState state,
- long lastHeartbeat,
- DatanodeStorageReport storageReport,
- List<DatanodePipeline> pipelines,
- int containers) {
- this.hostname = hostname;
- this.state = state;
- this.lastHeartbeat = lastHeartbeat;
- this.datanodeStorageReport = storageReport;
- this.pipelines = pipelines;
- this.containers = containers;
+ @XmlElement(name = "leaderCount")
+ private int leaderCount;
+
+ private DatanodeMetadata(Builder builder) {
+ this.hostname = builder.hostname;
+ this.state = builder.state;
+ this.lastHeartbeat = builder.lastHeartbeat;
+ this.datanodeStorageReport = builder.datanodeStorageReport;
+ this.pipelines = builder.pipelines;
+ this.containers = builder.containers;
+ this.leaderCount = builder.leaderCount;
}
public String getHostname() {
@@ -85,4 +85,84 @@
public int getContainers() {
return containers;
}
+
+ public int getLeaderCount() {
+ return leaderCount;
+ }
+
+ /**
+ * Returns new builder class that builds a DatanodeMetadata.
+ *
+ * @return Builder
+ */
+ public static Builder newBuilder() {
+ return new Builder();
+ }
+
+ /**
+ * Builder for DatanodeMetadata.
+ */
+ @SuppressWarnings("checkstyle:hiddenfield")
+ public static final class Builder {
+ private String hostname;
+ private NodeState state;
+ private long lastHeartbeat;
+ private DatanodeStorageReport datanodeStorageReport;
+ private List<DatanodePipeline> pipelines;
+ private int containers;
+ private int leaderCount;
+
+ public Builder() {
+ this.containers = 0;
+ this.leaderCount = 0;
+ }
+
+ public Builder withHostname(String hostname) {
+ this.hostname = hostname;
+ return this;
+ }
+
+ public Builder withState(NodeState state) {
+ this.state = state;
+ return this;
+ }
+
+ public Builder withLastHeartbeat(long lastHeartbeat) {
+ this.lastHeartbeat = lastHeartbeat;
+ return this;
+ }
+
+ public Builder withDatanodeStorageReport(DatanodeStorageReport
+ datanodeStorageReport) {
+ this.datanodeStorageReport = datanodeStorageReport;
+ return this;
+ }
+
+ public Builder withPipelines(List<DatanodePipeline> pipelines) {
+ this.pipelines = pipelines;
+ return this;
+ }
+
+ public Builder withContainers(int containers) {
+ this.containers = containers;
+ return this;
+ }
+
+ public Builder withLeaderCount(int leaderCount) {
+ this.leaderCount = leaderCount;
+ return this;
+ }
+
+ /**
+ * Constructs DatanodeMetadata.
+ *
+ * @return instance of DatanodeMetadata.
+ */
+ public DatanodeMetadata build() {
+ Preconditions.checkNotNull(hostname);
+ Preconditions.checkNotNull(state);
+
+ return new DatanodeMetadata(this);
+ }
+ }
}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DatanodePipeline.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DatanodePipeline.java
index 6cc663b..cc38cd5 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DatanodePipeline.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DatanodePipeline.java
@@ -26,12 +26,14 @@
private UUID pipelineID;
private String replicationType;
private int replicationFactor;
+ private String leaderNode;
public DatanodePipeline(UUID pipelineID, String replicationType,
- int replicationFactor) {
+ int replicationFactor, String leaderNode) {
this.pipelineID = pipelineID;
this.replicationType = replicationType;
this.replicationFactor = replicationFactor;
+ this.leaderNode = leaderNode;
}
public UUID getPipelineID() {
@@ -45,4 +47,8 @@
public int getReplicationFactor() {
return replicationFactor;
}
+
+ public String getLeaderNode() {
+ return leaderNode;
+ }
}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeysResponse.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeysResponse.java
index 59e23ad..5b05975 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeysResponse.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeysResponse.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.ozone.recon.api.types;
import com.fasterxml.jackson.annotation.JsonProperty;
-import java.util.ArrayList;
import java.util.Collection;
/**
@@ -26,57 +25,27 @@
*/
public class KeysResponse {
/**
- * Contains a map with total count of keys inside the given container and a
- * list of keys with metadata.
+ * Total count of the keys.
*/
- @JsonProperty("data")
- private KeysResponseData keysResponseData;
-
- public KeysResponse() {
- this(0, new ArrayList<>());
- }
-
- public KeysResponse(long totalCount,
- Collection<KeyMetadata> keys) {
- this.keysResponseData =
- new KeysResponseData(totalCount, keys);
- }
-
- public KeysResponseData getKeysResponseData() {
- return keysResponseData;
- }
-
- public void setKeysResponseData(KeysResponseData keysResponseData) {
- this.keysResponseData = keysResponseData;
- }
+ @JsonProperty("totalCount")
+ private long totalCount;
/**
- * Class that encapsulates the data presented in Keys API Response.
+ * An array of keys.
*/
- public static class KeysResponseData {
- /**
- * Total count of the keys.
- */
- @JsonProperty("totalCount")
- private long totalCount;
+ @JsonProperty("keys")
+ private Collection<KeyMetadata> keys;
- /**
- * An array of keys.
- */
- @JsonProperty("keys")
- private Collection<KeyMetadata> keys;
+ public KeysResponse(long totalCount, Collection<KeyMetadata> keys) {
+ this.totalCount = totalCount;
+ this.keys = keys;
+ }
- KeysResponseData(long totalCount, Collection<KeyMetadata> keys) {
- this.totalCount = totalCount;
- this.keys = keys;
- }
+ public long getTotalCount() {
+ return totalCount;
+ }
- public long getTotalCount() {
- return totalCount;
- }
-
- public Collection<KeyMetadata> getKeys() {
- return keys;
- }
+ public Collection<KeyMetadata> getKeys() {
+ return keys;
}
}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/MissingContainerMetadata.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/MissingContainerMetadata.java
index f24bc57..3eff647 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/MissingContainerMetadata.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/MissingContainerMetadata.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.ozone.recon.api.types;
+import org.hadoop.ozone.recon.schema.tables.pojos.ContainerHistory;
+
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
@@ -41,17 +43,17 @@
@XmlElement(name = "pipelineID")
private UUID pipelineID;
- @XmlElement(name = "datanodes")
- private List<String> datanodes;
+ @XmlElement(name = "replicas")
+ private List<ContainerHistory> replicas;
public MissingContainerMetadata(long containerID, long missingSince,
long keys, UUID pipelineID,
- List<String> datanodes) {
+ List<ContainerHistory> replicas) {
this.containerID = containerID;
this.missingSince = missingSince;
this.keys = keys;
this.pipelineID = pipelineID;
- this.datanodes = datanodes;
+ this.replicas = replicas;
}
public long getContainerID() {
@@ -62,8 +64,8 @@
return keys;
}
- public List<String> getDatanodes() {
- return datanodes;
+ public List<ContainerHistory> getReplicas() {
+ return replicas;
}
public long getMissingSince() {
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/MissingContainerTask.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/MissingContainerTask.java
index 6db2025..9c1a250 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/MissingContainerTask.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/MissingContainerTask.java
@@ -19,20 +19,19 @@
package org.apache.hadoop.ozone.recon.fsck;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
-import javax.inject.Inject;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerManager;
import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
import org.apache.hadoop.hdds.scm.container.ContainerReplica;
-import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
+import org.apache.hadoop.ozone.recon.persistence.ContainerSchemaManager;
import org.apache.hadoop.ozone.recon.scm.ReconScmTask;
+import org.apache.hadoop.ozone.recon.tasks.ReconTaskConfig;
import org.apache.hadoop.util.Time;
-import org.hadoop.ozone.recon.schema.tables.daos.MissingContainersDao;
import org.hadoop.ozone.recon.schema.tables.daos.ReconTaskStatusDao;
-import org.hadoop.ozone.recon.schema.tables.pojos.MissingContainers;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.CollectionUtils;
@@ -47,17 +46,19 @@
LoggerFactory.getLogger(MissingContainerTask.class);
private ContainerManager containerManager;
- private MissingContainersDao missingContainersDao;
- private static final long INTERVAL = 5 * 60 * 1000L;
+ private ContainerSchemaManager containerSchemaManager;
+ private final long interval;
- @Inject
public MissingContainerTask(
- OzoneStorageContainerManager ozoneStorageContainerManager,
+ ContainerManager containerManager,
ReconTaskStatusDao reconTaskStatusDao,
- MissingContainersDao missingContainersDao) {
+ ContainerSchemaManager containerSchemaManager,
+ ReconTaskConfig reconTaskConfig) {
super(reconTaskStatusDao);
- this.missingContainersDao = missingContainersDao;
- this.containerManager = ozoneStorageContainerManager.getContainerManager();
+ this.containerSchemaManager = containerSchemaManager;
+ this.containerManager = containerManager;
+ this.interval = TimeUnit.SECONDS.toMillis(
+ reconTaskConfig.getMissingContainerTaskInterval());
}
public synchronized void run() {
@@ -73,7 +74,7 @@
LOG.info("Missing Container task Thread took {} milliseconds for" +
" processing {} containers.", Time.monotonicNow() - start,
containerIds.size());
- wait(INTERVAL);
+ wait(interval);
}
} catch (Throwable t) {
LOG.error("Exception in Missing Container task Thread.", t);
@@ -89,17 +90,20 @@
boolean isAllUnhealthy =
containerReplicas.stream().allMatch(replica ->
replica.getState().equals(State.UNHEALTHY));
+ boolean isMissingContainer =
+ containerSchemaManager.isMissingContainer(containerID.getId());
if (CollectionUtils.isEmpty(containerReplicas) || isAllUnhealthy) {
- if (!missingContainersDao.existsById(containerID.getId())) {
- LOG.info("Found a missing container with ID {}. Adding it to the " +
- "database", containerID.getId());
- MissingContainers newRecord =
- new MissingContainers(containerID.getId(), currentTime);
- missingContainersDao.insert(newRecord);
+ if (!isMissingContainer) {
+ LOG.info("Found a missing container with ID {}.",
+ containerID.getId());
+ containerSchemaManager.addMissingContainer(containerID.getId(),
+ currentTime);
}
} else {
- if (missingContainersDao.existsById(containerID.getId())) {
- missingContainersDao.deleteById(containerID.getId());
+ if (isMissingContainer) {
+ LOG.info("Missing container with ID {} is no longer missing.",
+ containerID.getId());
+ containerSchemaManager.deleteMissingContainer(containerID.getId());
}
}
} catch (ContainerNotFoundException e) {
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/metrics/OzoneManagerSyncMetrics.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/metrics/OzoneManagerSyncMetrics.java
new file mode 100644
index 0000000..1e3fa5a
--- /dev/null
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/metrics/OzoneManagerSyncMetrics.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.metrics;
+
+import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableGaugeFloat;
+import org.apache.hadoop.metrics2.lib.MutableRate;
+import org.apache.hadoop.ozone.OzoneConsts;
+
+/**
+ * Class for tracking metrics related to Ozone manager sync operations.
+ */
+@InterfaceAudience.Private
+@Metrics(about = "Recon OzoneManagerSync Metrics", context = OzoneConsts.OZONE)
+public final class OzoneManagerSyncMetrics {
+
+ private static final String SOURCE_NAME =
+ OzoneManagerSyncMetrics.class.getSimpleName();
+
+ private OzoneManagerSyncMetrics() {
+ }
+
+ public static OzoneManagerSyncMetrics create() {
+ MetricsSystem ms = DefaultMetricsSystem.instance();
+ return ms.register(SOURCE_NAME,
+ "Recon Ozone Manager Sync Metrics",
+ new OzoneManagerSyncMetrics());
+ }
+
+ public void unRegister() {
+ MetricsSystem ms = DefaultMetricsSystem.instance();
+ ms.unregisterSource(SOURCE_NAME);
+ }
+
+ @Metric(about = "Number of OM snapshot requests made by Recon.")
+ private MutableCounterLong numSnapshotRequests;
+
+ @Metric(about = "Number of OM snapshot requests that failed.")
+ private MutableCounterLong numSnapshotRequestsFailed;
+
+ @Metric(about = "OM snapshot request latency")
+ private MutableRate snapshotRequestLatency;
+
+ @Metric(about = "Number of OM delta requests made by Recon that had " +
+ "at least 1 update in the response.")
+ private MutableCounterLong numNonZeroDeltaRequests;
+
+ @Metric(about = "Number of OM delta requests that failed.")
+ private MutableCounterLong numDeltaRequestsFailed;
+
+ @Metric(about = "Total number of updates got through OM delta request")
+ private MutableCounterLong numUpdatesInDeltaTotal;
+
+ @Metric(about = "Average number of updates got per OM delta request")
+ private MutableGaugeFloat averageNumUpdatesInDeltaRequest;
+
+ public void incrNumSnapshotRequests() {
+ this.numSnapshotRequests.incr();
+ }
+
+ public void incrNumSnapshotRequestsFailed() {
+ this.numSnapshotRequestsFailed.incr();
+ }
+
+ public void updateSnapshotRequestLatency(long time) {
+ this.snapshotRequestLatency.add(time);
+ }
+
+ public void incrNumDeltaRequestsFailed() {
+ this.numSnapshotRequestsFailed.incr();
+ }
+
+ public void incrNumUpdatesInDeltaTotal(long n) {
+ this.numUpdatesInDeltaTotal.incr(n);
+ this.numNonZeroDeltaRequests.incr();
+ setAverageNumUpdatesInDeltaRequest(
+ (float) this.numUpdatesInDeltaTotal.value() /
+ (float) this.numNonZeroDeltaRequests.value());
+ }
+
+ public void setAverageNumUpdatesInDeltaRequest(float avg) {
+ averageNumUpdatesInDeltaRequest.set(avg);
+ }
+
+ public MutableCounterLong getNumSnapshotRequests() {
+ return numSnapshotRequests;
+ }
+
+ public MutableCounterLong getNumSnapshotRequestsFailed() {
+ return numSnapshotRequestsFailed;
+ }
+
+ public MutableRate getSnapshotRequestLatency() {
+ return snapshotRequestLatency;
+ }
+
+ public MutableCounterLong getNumDeltaRequestsFailed() {
+ return numDeltaRequestsFailed;
+ }
+
+ public MutableCounterLong getNumUpdatesInDeltaTotal() {
+ return numUpdatesInDeltaTotal;
+ }
+
+ public MutableGaugeFloat getAverageNumUpdatesInDeltaRequest() {
+ return averageNumUpdatesInDeltaRequest;
+ }
+
+ public MutableCounterLong getNumNonZeroDeltaRequests() {
+ return numNonZeroDeltaRequests;
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/package-info.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/metrics/package-info.java
similarity index 88%
copy from hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/package-info.java
copy to hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/metrics/package-info.java
index 80c1985..9158594 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/package-info.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/metrics/package-info.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -15,7 +15,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.ozone.web.ozShell;
+
/**
- * Tests for ozone shell..
+ * This package contains Recon metrics related classes.
*/
+package org.apache.hadoop.ozone.recon.metrics;
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/ContainerSchemaManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/ContainerSchemaManager.java
new file mode 100644
index 0000000..6dc70a2
--- /dev/null
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/ContainerSchemaManager.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.recon.persistence;
+
+import static org.hadoop.ozone.recon.schema.tables.ContainerHistoryTable.CONTAINER_HISTORY;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import org.hadoop.ozone.recon.schema.ContainerSchemaDefinition;
+import org.hadoop.ozone.recon.schema.tables.daos.ContainerHistoryDao;
+import org.hadoop.ozone.recon.schema.tables.daos.MissingContainersDao;
+import org.hadoop.ozone.recon.schema.tables.pojos.ContainerHistory;
+import org.hadoop.ozone.recon.schema.tables.pojos.MissingContainers;
+import org.jooq.DSLContext;
+import org.jooq.Record2;
+import java.util.List;
+
+/**
+ * Provide a high level API to access the Container Schema.
+ */
+@Singleton
+public class ContainerSchemaManager {
+ private ContainerHistoryDao containerHistoryDao;
+ private MissingContainersDao missingContainersDao;
+ private ContainerSchemaDefinition containerSchemaDefinition;
+
+ @Inject
+ public ContainerSchemaManager(ContainerHistoryDao containerHistoryDao,
+ ContainerSchemaDefinition containerSchemaDefinition,
+ MissingContainersDao missingContainersDao) {
+ this.containerHistoryDao = containerHistoryDao;
+ this.missingContainersDao = missingContainersDao;
+ this.containerSchemaDefinition = containerSchemaDefinition;
+ }
+
+ public void addMissingContainer(long containerID, long time) {
+ MissingContainers record = new MissingContainers(containerID, time);
+ missingContainersDao.insert(record);
+ }
+
+ public List<MissingContainers> getAllMissingContainers() {
+ return missingContainersDao.findAll();
+ }
+
+ public boolean isMissingContainer(long containerID) {
+ return missingContainersDao.existsById(containerID);
+ }
+
+ public void deleteMissingContainer(long containerID) {
+ missingContainersDao.deleteById(containerID);
+ }
+
+ public void upsertContainerHistory(long containerID, String datanode,
+ long time) {
+ DSLContext dslContext = containerSchemaDefinition.getDSLContext();
+ Record2<Long, String> recordToFind =
+ dslContext.newRecord(
+ CONTAINER_HISTORY.CONTAINER_ID,
+ CONTAINER_HISTORY.DATANODE_HOST).value1(containerID).value2(datanode);
+ ContainerHistory newRecord = new ContainerHistory();
+ newRecord.setContainerId(containerID);
+ newRecord.setDatanodeHost(datanode);
+ newRecord.setLastReportTimestamp(time);
+ ContainerHistory record = containerHistoryDao.findById(recordToFind);
+ if (record != null) {
+ newRecord.setFirstReportTimestamp(record.getFirstReportTimestamp());
+ containerHistoryDao.update(newRecord);
+ } else {
+ newRecord.setFirstReportTimestamp(time);
+ containerHistoryDao.insert(newRecord);
+ }
+ }
+
+ public List<ContainerHistory> getAllContainerHistory(long containerID) {
+ return containerHistoryDao.fetchByContainerId(containerID);
+ }
+
+ public List<ContainerHistory> getLatestContainerHistory(long containerID,
+ int limit) {
+ DSLContext dslContext = containerSchemaDefinition.getDSLContext();
+ // Get container history sorted in descending order of last report timestamp
+ return dslContext.select()
+ .from(CONTAINER_HISTORY)
+ .where(CONTAINER_HISTORY.CONTAINER_ID.eq(containerID))
+ .orderBy(CONTAINER_HISTORY.LAST_REPORT_TIMESTAMP.desc())
+ .limit(limit)
+ .fetchInto(ContainerHistory.class);
+ }
+}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/DataSourceConfiguration.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/DataSourceConfiguration.java
index 54ef888..7e97c4f 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/DataSourceConfiguration.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/DataSourceConfiguration.java
@@ -66,12 +66,12 @@
/**
* Sets the maximum connection age (in seconds).
*/
- Integer getMaxConnectionAge();
+ long getMaxConnectionAge();
/**
* Sets the maximum idle connection age (in seconds).
*/
- Integer getMaxIdleConnectionAge();
+ long getMaxIdleConnectionAge();
/**
* Statement specific to database, usually SELECT 1.
@@ -81,5 +81,5 @@
/**
* How often to test idle connections for being active (in seconds).
*/
- Integer getIdleConnectionTestPeriod();
+ long getIdleConnectionTestPeriod();
}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/DefaultDataSourceProvider.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/DefaultDataSourceProvider.java
index b0b8847..42cde7d 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/DefaultDataSourceProvider.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/DefaultDataSourceProvider.java
@@ -20,7 +20,6 @@
import javax.sql.DataSource;
import org.apache.commons.lang3.StringUtils;
-import org.sqlite.SQLiteDataSource;
import com.google.inject.Inject;
import com.google.inject.Provider;
@@ -43,14 +42,14 @@
*/
@Override
public DataSource get() {
- if (StringUtils.contains(configuration.getJdbcUrl(), "sqlite")) {
- SQLiteDataSource ds = new SQLiteDataSource();
- ds.setUrl(configuration.getJdbcUrl());
- return ds;
+ String jdbcUrl = configuration.getJdbcUrl();
+ if (StringUtils.contains(jdbcUrl, "derby")) {
+ return new DerbyDataSourceProvider(configuration).get();
+ } else if (StringUtils.contains(jdbcUrl, "sqlite")) {
+ return new SqliteDataSourceProvider(configuration).get();
}
BoneCPDataSource cpDataSource = new BoneCPDataSource();
-
cpDataSource.setDriverClass(configuration.getDriverClass());
cpDataSource.setJdbcUrl(configuration.getJdbcUrl());
cpDataSource.setUsername(configuration.getUserName());
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/DerbyDataSourceProvider.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/DerbyDataSourceProvider.java
new file mode 100644
index 0000000..51678c0
--- /dev/null
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/DerbyDataSourceProvider.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.recon.persistence;
+
+import static org.hadoop.ozone.recon.codegen.JooqCodeGenerator.RECON_SCHEMA_NAME;
+import static org.hadoop.ozone.recon.codegen.SqlDbUtils.createNewDerbyDatabase;
+
+import javax.sql.DataSource;
+
+import org.apache.derby.jdbc.EmbeddedDataSource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+
+/**
+ * Provide a {@link javax.sql.DataSource} for the application.
+ */
+public class DerbyDataSourceProvider implements Provider<DataSource> {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DerbyDataSourceProvider.class);
+
+ private DataSourceConfiguration configuration;
+
+ @Inject
+ DerbyDataSourceProvider(DataSourceConfiguration configuration) {
+ this.configuration = configuration;
+ }
+
+ @Override
+ public DataSource get() {
+ String jdbcUrl = configuration.getJdbcUrl();
+ LOG.info("JDBC Url for Recon : {} ", jdbcUrl);
+ try {
+ createNewDerbyDatabase(jdbcUrl, RECON_SCHEMA_NAME);
+ } catch (Exception e) {
+ LOG.error("Error creating Recon Derby DB.", e);
+ }
+ EmbeddedDataSource dataSource = new EmbeddedDataSource();
+ dataSource.setDatabaseName(jdbcUrl.split(":")[2]);
+ dataSource.setUser(RECON_SCHEMA_NAME);
+ return dataSource;
+ }
+}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/JooqPersistenceModule.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/JooqPersistenceModule.java
index f7ab4a5..a28cdf2 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/JooqPersistenceModule.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/JooqPersistenceModule.java
@@ -45,7 +45,7 @@
public class JooqPersistenceModule extends AbstractModule {
private Provider<DataSourceConfiguration> configurationProvider;
- public static final SQLDialect DEFAULT_DIALECT = SQLDialect.SQLITE;
+ public static final SQLDialect DEFAULT_DIALECT = SQLDialect.DERBY;
public JooqPersistenceModule(
Provider<DataSourceConfiguration> configurationProvider) {
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/SqliteDataSourceProvider.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/SqliteDataSourceProvider.java
new file mode 100644
index 0000000..897f8be
--- /dev/null
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/SqliteDataSourceProvider.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.persistence;
+
+import javax.sql.DataSource;
+
+import org.sqlite.SQLiteDataSource;
+
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+
+/**
+ * Provide a {@link javax.sql.DataSource} for the application.
+ */
+public class SqliteDataSourceProvider implements Provider<DataSource> {
+
+ private DataSourceConfiguration configuration;
+
+ @Inject
+ public SqliteDataSourceProvider(DataSourceConfiguration configuration) {
+ this.configuration = configuration;
+ }
+
+ /**
+ * Create a pooled datasource for the application.
+ * <p>
+ * Default sqlite database does not work with a connection pool, actually
+ * most embedded databases do not, hence returning native implementation for
+ * default db.
+ */
+ @Override
+ public DataSource get() {
+ SQLiteDataSource ds = new SQLiteDataSource();
+ ds.setUrl(configuration.getJdbcUrl());
+ return ds;
+ }
+}
\ No newline at end of file
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOMMetadataManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOMMetadataManager.java
index 8c7dca9..a6104bf 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOMMetadataManager.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOMMetadataManager.java
@@ -41,4 +41,10 @@
* Database.
*/
long getLastSequenceNumberFromDB();
+
+ /**
+ * Check if OM tables are initialized.
+ * @return true if OM Tables are initialized, otherwise false.
+ */
+ boolean isOmTablesInitialized();
}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOmMetadataManagerImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOmMetadataManagerImpl.java
index e60d0dd..fc5cead 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOmMetadataManagerImpl.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOmMetadataManagerImpl.java
@@ -51,6 +51,7 @@
private OzoneConfiguration ozoneConfiguration;
private ReconUtils reconUtils;
+ private boolean omTablesInitialized = false;
@Inject
public ReconOmMetadataManagerImpl(OzoneConfiguration configuration,
@@ -94,6 +95,7 @@
}
if (getStore() != null) {
initializeOmTables();
+ omTablesInitialized = true;
}
}
@@ -120,4 +122,12 @@
}
}
+ /**
+ * Check if OM tables are initialized.
+ * @return true if OM Tables are initialized, otherwise false.
+ */
+ @Override
+ public boolean isOmTablesInitialized() {
+ return omTablesInitialized;
+ }
}
\ No newline at end of file
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/PipelineSyncTask.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/PipelineSyncTask.java
index 80de0ae..ecd1b45 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/PipelineSyncTask.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/PipelineSyncTask.java
@@ -19,12 +19,11 @@
package org.apache.hadoop.ozone.recon.scm;
import java.util.List;
-
-import javax.inject.Inject;
+import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
-import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider;
+import org.apache.hadoop.ozone.recon.tasks.ReconTaskConfig;
import org.apache.hadoop.util.Time;
import org.hadoop.ozone.recon.schema.tables.daos.ReconTaskStatusDao;
import org.slf4j.Logger;
@@ -41,17 +40,17 @@
private StorageContainerServiceProvider scmClient;
private ReconPipelineManager reconPipelineManager;
- private static final long INTERVAL = 10 * 60 * 1000L;
+ private final long interval;
- @Inject
- public PipelineSyncTask(
- OzoneStorageContainerManager storageContainerManager,
+ public PipelineSyncTask(ReconPipelineManager pipelineManager,
StorageContainerServiceProvider scmClient,
- ReconTaskStatusDao reconTaskStatusDao) {
+ ReconTaskStatusDao reconTaskStatusDao,
+ ReconTaskConfig reconTaskConfig) {
super(reconTaskStatusDao);
this.scmClient = scmClient;
- this.reconPipelineManager = (ReconPipelineManager)
- storageContainerManager.getPipelineManager();
+ this.reconPipelineManager = pipelineManager;
+ this.interval = TimeUnit.SECONDS.toMillis(
+ reconTaskConfig.getPipelineSyncTaskInterval());
}
@Override
@@ -64,7 +63,7 @@
LOG.info("Pipeline sync Thread took {} milliseconds.",
Time.monotonicNow() - start);
recordSingleRunCompletion();
- wait(INTERVAL);
+ wait(interval);
}
} catch (Throwable t) {
LOG.error("Exception in Pipeline sync Thread.", t);
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java
index 06ee5ce..72d1548 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java
@@ -18,20 +18,22 @@
package org.apache.hadoop.ozone.recon.scm;
-import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_SCM_CONTAINER_DB;
-
-import java.io.File;
import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
import org.apache.hadoop.hdds.scm.container.SCMContainerManager;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
-import org.apache.hadoop.ozone.recon.ReconUtils;
+import org.apache.hadoop.hdds.utils.db.BatchOperationHandler;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.ozone.recon.persistence.ContainerSchemaManager;
import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,6 +45,7 @@
private static final Logger LOG =
LoggerFactory.getLogger(ReconContainerManager.class);
private StorageContainerServiceProvider scmClient;
+ private ContainerSchemaManager containerSchemaManager;
/**
* Constructs a mapping class that creates mapping between container names
@@ -52,31 +55,29 @@
* CacheSize is specified
* in MB.
*
- * @param conf - {@link Configuration}
- * @param pipelineManager - {@link PipelineManager}
* @throws IOException on Failure.
*/
public ReconContainerManager(
- Configuration conf, PipelineManager pipelineManager,
- StorageContainerServiceProvider scm) throws IOException {
- super(conf, pipelineManager);
+ ConfigurationSource conf,
+ Table<ContainerID, ContainerInfo> containerStore,
+ BatchOperationHandler batchHandler,
+ PipelineManager pipelineManager,
+ StorageContainerServiceProvider scm,
+ ContainerSchemaManager containerSchemaManager) throws IOException {
+ super(conf, containerStore, batchHandler, pipelineManager);
this.scmClient = scm;
- }
-
- @Override
- protected File getContainerDBPath(Configuration conf) {
- File metaDir = ReconUtils.getReconScmDbDir(conf);
- return new File(metaDir, RECON_SCM_CONTAINER_DB);
+ this.containerSchemaManager = containerSchemaManager;
}
/**
* Check and add new container if not already present in Recon.
- * @param containerID containerID to check.
+ *
+ * @param containerID containerID to check.
* @param datanodeDetails Datanode from where we got this container.
* @throws IOException on Error.
*/
public void checkAndAddNewContainer(ContainerID containerID,
- DatanodeDetails datanodeDetails)
+ DatanodeDetails datanodeDetails)
throws IOException {
if (!exists(containerID)) {
LOG.info("New container {} got from {}.", containerID,
@@ -128,4 +129,26 @@
getLock().unlock();
}
}
+
+ /**
+ * Add a container Replica for given DataNode.
+ *
+ * @param containerID
+ * @param replica
+ */
+ @Override
+ public void updateContainerReplica(ContainerID containerID,
+ ContainerReplica replica)
+ throws ContainerNotFoundException {
+ super.updateContainerReplica(containerID, replica);
+ // Update container_history table
+ long currentTime = System.currentTimeMillis();
+ String datanodeHost = replica.getDatanodeDetails().getHostName();
+ containerSchemaManager.upsertContainerHistory(containerID.getId(),
+ datanodeHost, currentTime);
+ }
+
+ public ContainerSchemaManager getContainerSchemaManager() {
+ return containerSchemaManager;
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/bucket/package-info.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconDBDefinition.java
similarity index 64%
copy from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/bucket/package-info.java
copy to hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconDBDefinition.java
index f484ecc..bcfe060 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/bucket/package-info.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconDBDefinition.java
@@ -16,9 +16,23 @@
* limitations under the License.
*
*/
+package org.apache.hadoop.ozone.recon.scm;
+
+import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition;
+import org.apache.hadoop.ozone.recon.ReconServerConfigKeys;
/**
- * Package contains classes related to s3 bucket responses.
+ * SCM db file for ozone.
*/
-package org.apache.hadoop.ozone.om.response.s3.bucket;
+public class ReconDBDefinition extends SCMDBDefinition {
+ @Override
+ public String getName() {
+ return "recon-scm.db";
+ }
+
+ @Override
+ public String getLocationConfigKey() {
+ return ReconServerConfigKeys.OZONE_RECON_SCM_DB_DIR;
+ }
+}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconDatanodeProtocolServer.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconDatanodeProtocolServer.java
index 6d58931..a9cde11 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconDatanodeProtocolServer.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconDatanodeProtocolServer.java
@@ -32,6 +32,7 @@
import org.apache.hadoop.ozone.protocolPB.ReconDatanodeProtocolPB;
import org.apache.hadoop.security.authorize.PolicyProvider;
+import com.google.protobuf.ProtocolMessageEnum;
import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_DATANODE_ADDRESS_KEY;
/**
@@ -48,7 +49,8 @@
}
@Override
- public ProtocolMessageMetrics getProtocolMessageMetrics() {
+ public ProtocolMessageMetrics<ProtocolMessageEnum>
+ getProtocolMessageMetrics() {
return ProtocolMessageMetrics
.create("ReconDatanodeProtocol", "Recon Datanode protocol",
StorageContainerDatanodeProtocolProtos.Type.values());
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java
index 9a3d518..60e8a06 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java
@@ -18,11 +18,6 @@
package org.apache.hadoop.ozone.recon.scm;
-import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.reregisterCommand;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_DEFAULT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_MB;
-import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_SCM_NODE_DB;
-
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
@@ -31,8 +26,8 @@
import java.util.Set;
import java.util.UUID;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.StringUtils;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -48,10 +43,14 @@
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.hadoop.ozone.recon.ReconUtils;
import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import com.google.common.collect.ImmutableSet;
+import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.reregisterCommand;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_MB;
+import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_SCM_NODE_DB;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Recon SCM's Node manager that includes persistence.
@@ -117,7 +116,7 @@
LOG.info("Adding new node {} to Node DB.", datanodeDetails.getUuid());
}
- protected File getNodeDBPath(Configuration conf) {
+ protected File getNodeDBPath(ConfigurationSource conf) {
File metaDir = ReconUtils.getReconScmDbDir(conf);
return new File(metaDir, RECON_SCM_NODE_DB);
}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineFactory.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineFactory.java
index 1ab037a..2813301 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineFactory.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineFactory.java
@@ -39,7 +39,7 @@
setProviders(new DefaultedMap(reconMockPipelineProvider));
}
- static class ReconPipelineProvider implements PipelineProvider {
+ static class ReconPipelineProvider extends PipelineProvider {
@Override
public Pipeline create(HddsProtos.ReplicationFactor factor){
@@ -63,7 +63,7 @@
@Override
public void shutdown() {
-
+ // Do nothing
}
}
}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineManager.java
index 2405ad2..a8dd3c9 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineManager.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineManager.java
@@ -18,15 +18,11 @@
package org.apache.hadoop.ozone.recon.scm;
-import static org.apache.hadoop.hdds.scm.pipeline.Pipeline.PipelineState.CLOSED;
-import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_SCM_PIPELINE_DB;
-
-import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.stream.Collectors;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
@@ -34,11 +30,14 @@
import org.apache.hadoop.hdds.scm.pipeline.PipelineStateManager;
import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
import org.apache.hadoop.hdds.server.events.EventPublisher;
-import org.apache.hadoop.ozone.recon.ReconUtils;
+import org.apache.hadoop.hdds.utils.db.Table;
+
+import com.google.common.annotations.VisibleForTesting;
+import static org.apache.hadoop.hdds.scm.pipeline.Pipeline.PipelineState.CLOSED;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.annotations.VisibleForTesting;
+
/**
* Recon's overriding implementation of SCM's Pipeline Manager.
@@ -48,21 +47,17 @@
private static final Logger LOG =
LoggerFactory.getLogger(ReconPipelineManager.class);
- public ReconPipelineManager(Configuration conf,
- NodeManager nodeManager,
- EventPublisher eventPublisher)
+ public ReconPipelineManager(ConfigurationSource conf,
+ NodeManager nodeManager,
+ Table<PipelineID, Pipeline> pipelineStore,
+ EventPublisher eventPublisher)
throws IOException {
- super(conf, nodeManager, eventPublisher, new PipelineStateManager(),
+ super(conf, nodeManager, pipelineStore, eventPublisher,
+ new PipelineStateManager(),
new ReconPipelineFactory());
initializePipelineState();
}
-
- @Override
- protected File getPipelineDBPath(Configuration conf) {
- File metaDir = ReconUtils.getReconScmDbDir(conf);
- return new File(metaDir, RECON_SCM_PIPELINE_DB);
- }
-
+
@Override
public void triggerPipelineCreation() {
// Don't do anything in Recon.
@@ -148,8 +143,7 @@
void addPipeline(Pipeline pipeline) throws IOException {
getLock().writeLock().lock();
try {
- getPipelineStore().put(pipeline.getId().getProtobuf().toByteArray(),
- pipeline.getProtobufMessage().toByteArray());
+ getPipelineStore().put(pipeline.getId(), pipeline);
getStateManager().addPipeline(pipeline);
getNodeManager().addPipeline(pipeline);
} finally {
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineReportHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineReportHandler.java
index e3b8c02..246d9ba 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineReportHandler.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineReportHandler.java
@@ -20,7 +20,7 @@
import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReport;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
@@ -46,7 +46,7 @@
public ReconPipelineReportHandler(SafeModeManager scmSafeModeManager,
PipelineManager pipelineManager,
- Configuration conf,
+ ConfigurationSource conf,
StorageContainerServiceProvider scmServiceProvider) {
super(scmSafeModeManager, pipelineManager, conf);
this.scmServiceProvider = scmServiceProvider;
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconScmTask.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconScmTask.java
index 27d9892..df21c21 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconScmTask.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconScmTask.java
@@ -23,8 +23,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.inject.Inject;
-
/**
* Any background task that keeps SCM's metadata up to date.
*/
@@ -35,12 +33,11 @@
private ReconTaskStatusDao reconTaskStatusDao;
private volatile boolean running;
- @Inject
- public ReconScmTask(ReconTaskStatusDao reconTaskStatusDao) {
+ protected ReconScmTask(ReconTaskStatusDao reconTaskStatusDao) {
this.reconTaskStatusDao = reconTaskStatusDao;
}
- public void register() {
+ private void register() {
String taskName = getTaskName();
if (!reconTaskStatusDao.existsById(taskName)) {
ReconTaskStatus reconTaskStatusRecord = new ReconTaskStatus(
@@ -54,6 +51,7 @@
* Start underlying start thread.
*/
public synchronized void start() {
+ register();
if (!isRunning()) {
LOG.info("Starting {} Thread.", getTaskName());
running = true;
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
index 9cb6a31..d6af3a8 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
@@ -18,9 +18,6 @@
package org.apache.hadoop.ozone.recon.scm;
-import static org.apache.hadoop.hdds.recon.ReconConfigKeys.RECON_SCM_CONFIG_PREFIX;
-import static org.apache.hadoop.hdds.scm.server.StorageContainerManager.buildRpcServerStartMessage;
-
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.HashSet;
@@ -28,8 +25,6 @@
import java.util.Map;
import java.util.Set;
-
-import com.google.inject.Inject;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.block.BlockManager;
import org.apache.hadoop.hdds.scm.container.CloseContainerEventHandler;
@@ -49,13 +44,19 @@
import org.apache.hadoop.hdds.scm.pipeline.PipelineActionHandler;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.scm.safemode.SafeModeManager;
-import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
+import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.recon.fsck.MissingContainerTask;
+import org.apache.hadoop.ozone.recon.persistence.ContainerSchemaManager;
import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider;
-import org.hadoop.ozone.recon.schema.tables.daos.MissingContainersDao;
+import org.apache.hadoop.ozone.recon.tasks.ReconTaskConfig;
+import com.google.inject.Inject;
+import static org.apache.hadoop.hdds.recon.ReconConfigKeys.RECON_SCM_CONFIG_PREFIX;
+import static org.apache.hadoop.hdds.scm.server.StorageContainerManager.buildRpcServerStartMessage;
import org.hadoop.ozone.recon.schema.tables.daos.ReconTaskStatusDao;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -73,6 +74,7 @@
private final ReconDatanodeProtocolServer datanodeProtocolServer;
private final EventQueue eventQueue;
private final SCMStorageConfig scmStorageConfig;
+ private final DBStore dbStore;
private ReconNodeManager nodeManager;
private ReconPipelineManager pipelineManager;
@@ -84,22 +86,33 @@
@Inject
public ReconStorageContainerManagerFacade(OzoneConfiguration conf,
StorageContainerServiceProvider scmServiceProvider,
- MissingContainersDao missingContainersDao,
- ReconTaskStatusDao reconTaskStatusDao)
+ ReconTaskStatusDao reconTaskStatusDao,
+ ContainerSchemaManager containerSchemaManager)
throws IOException {
this.eventQueue = new EventQueue();
eventQueue.setSilent(true);
this.ozoneConfiguration = getReconScmConfiguration(conf);
this.scmStorageConfig = new ReconStorageConfig(conf);
this.clusterMap = new NetworkTopologyImpl(conf);
+ dbStore = DBStoreBuilder
+ .createDBStore(ozoneConfiguration, new ReconDBDefinition());
+
this.nodeManager =
new ReconNodeManager(conf, scmStorageConfig, eventQueue, clusterMap);
this.datanodeProtocolServer = new ReconDatanodeProtocolServer(
conf, this, eventQueue);
this.pipelineManager =
- new ReconPipelineManager(conf, nodeManager, eventQueue);
- this.containerManager = new ReconContainerManager(conf, pipelineManager,
- scmServiceProvider);
+
+ new ReconPipelineManager(conf,
+ nodeManager,
+ ReconDBDefinition.PIPELINES.getTable(dbStore),
+ eventQueue);
+ this.containerManager = new ReconContainerManager(conf,
+ ReconDBDefinition.CONTAINERS.getTable(dbStore),
+ dbStore,
+ pipelineManager,
+ scmServiceProvider,
+ containerSchemaManager);
this.scmServiceProvider = scmServiceProvider;
NodeReportHandler nodeReportHandler =
@@ -141,19 +154,21 @@
eventQueue.addHandler(SCMEvents.CLOSE_CONTAINER, closeContainerHandler);
eventQueue.addHandler(SCMEvents.NEW_NODE, newNodeHandler);
+ ReconTaskConfig reconTaskConfig = conf.getObject(ReconTaskConfig.class);
reconScmTasks.add(new PipelineSyncTask(
- this,
+ pipelineManager,
scmServiceProvider,
- reconTaskStatusDao));
- reconScmTasks.add(new MissingContainerTask(
- this,
reconTaskStatusDao,
- missingContainersDao));
- reconScmTasks.forEach(ReconScmTask::register);
+ reconTaskConfig));
+ reconScmTasks.add(new MissingContainerTask(
+ containerManager,
+ reconTaskStatusDao,
+ containerSchemaManager,
+ reconTaskConfig));
}
/**
- * For every config key which is prefixed by 'recon.scm', create a new
+ * For every config key which is prefixed by 'recon.scmconfig', create a new
* config key without the prefix keeping the same value.
* For example, if recon.scm.a.b. = xyz, we add a new config like
* a.b.c = xyz. This is done to override Recon's passive SCM configs if
@@ -214,6 +229,11 @@
IOUtils.cleanupWithLogger(LOG, nodeManager);
IOUtils.cleanupWithLogger(LOG, containerManager);
IOUtils.cleanupWithLogger(LOG, pipelineManager);
+ try {
+ dbStore.close();
+ } catch (Exception e) {
+ LOG.error("Can't close dbStore ", e);
+ }
}
public ReconDatanodeProtocolServer getDatanodeProtocolServer() {
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java
index 11f8bfe..30c397b 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java
@@ -42,13 +42,13 @@
import org.apache.hadoop.ozone.recon.ReconUtils;
import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
+import org.apache.hadoop.ozone.recon.persistence.ContainerSchemaManager;
import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider;
import org.apache.hadoop.hdds.utils.db.DBStore;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.Table.KeyValue;
import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.hadoop.ozone.recon.schema.tables.daos.GlobalStatsDao;
-import org.hadoop.ozone.recon.schema.tables.daos.MissingContainersDao;
import org.hadoop.ozone.recon.schema.tables.pojos.GlobalStats;
import org.hadoop.ozone.recon.schema.tables.pojos.MissingContainers;
import org.jooq.Configuration;
@@ -70,6 +70,9 @@
private GlobalStatsDao globalStatsDao;
@Inject
+ private ContainerSchemaManager containerSchemaManager;
+
+ @Inject
private OzoneConfiguration configuration;
@Inject
@@ -82,9 +85,6 @@
private ReconUtils reconUtils;
@Inject
- private MissingContainersDao missingContainersDao;
-
- @Inject
public ContainerDBServiceProviderImpl(DBStore dbStore,
Configuration sqlConfiguration) {
containerDbStore = dbStore;
@@ -359,7 +359,7 @@
}
public List<MissingContainers> getMissingContainers() {
- return missingContainersDao.findAll();
+ return containerSchemaManager.getAllMissingContainers();
}
@Override
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java
index 94c0f48..94c9520 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java
@@ -18,8 +18,47 @@
package org.apache.hadoop.ozone.recon.spi.impl;
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.hdfs.web.URLConnectionFactory;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.server.http.HttpConfig;
+import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
+import org.apache.hadoop.hdds.utils.db.RDBBatchOperation;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDBCheckpoint;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.helpers.DBUpdates;
+import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DBUpdatesRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort.Type;
+import org.apache.hadoop.ozone.recon.ReconUtils;
+import org.apache.hadoop.ozone.recon.metrics.OzoneManagerSyncMetrics;
+import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
+import org.apache.hadoop.ozone.recon.spi.OzoneManagerServiceProvider;
+import org.apache.hadoop.ozone.recon.tasks.OMDBUpdatesHandler;
+import org.apache.hadoop.ozone.recon.tasks.OMUpdateEventBatch;
+import org.apache.hadoop.ozone.recon.tasks.ReconTaskController;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.util.Time;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.io.FileUtils;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_FLUSH;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OM_DB_CHECKPOINT_HTTP_ENDPOINT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_AUTH_TYPE;
import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_OM_SNAPSHOT_DB;
import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_OM_SNAPSHOT_DB_DIR;
import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.RECON_OM_CONNECTION_REQUEST_TIMEOUT;
@@ -31,46 +70,7 @@
import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.RECON_OM_SNAPSHOT_TASK_INITIAL_DELAY_DEFAULT;
import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.RECON_OM_SNAPSHOT_TASK_INTERVAL;
import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.RECON_OM_SNAPSHOT_TASK_INTERVAL_DEFAULT;
-import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.RECON_OM_SOCKET_TIMEOUT;
-import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.RECON_OM_SOCKET_TIMEOUT_DEFAULT;
import static org.apache.ratis.proto.RaftProtos.RaftPeerRole.LEADER;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.List;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import javax.inject.Inject;
-import javax.inject.Singleton;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.server.http.HttpConfig;
-import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
-import org.apache.hadoop.hdds.utils.db.DBUpdatesWrapper;
-import org.apache.hadoop.hdds.utils.db.RDBBatchOperation;
-import org.apache.hadoop.hdds.utils.db.RDBStore;
-import org.apache.hadoop.hdds.utils.db.RocksDBCheckpoint;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
-import org.apache.hadoop.ozone.om.OMMetadataManager;
-import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DBUpdatesRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort.Type;
-import org.apache.hadoop.ozone.recon.ReconUtils;
-import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
-import org.apache.hadoop.ozone.recon.spi.OzoneManagerServiceProvider;
-import org.apache.hadoop.ozone.recon.tasks.OMDBUpdatesHandler;
-import org.apache.hadoop.ozone.recon.tasks.OMUpdateEventBatch;
-import org.apache.hadoop.ozone.recon.tasks.ReconTaskController;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
import org.hadoop.ozone.recon.schema.tables.daos.ReconTaskStatusDao;
import org.hadoop.ozone.recon.schema.tables.pojos.ReconTaskStatus;
import org.rocksdb.RocksDB;
@@ -80,8 +80,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.annotations.VisibleForTesting;
-
/**
* Implementation of the OzoneManager Service provider.
*/
@@ -91,8 +89,8 @@
private static final Logger LOG =
LoggerFactory.getLogger(OzoneManagerServiceProviderImpl.class);
+ private URLConnectionFactory connectionFactory;
- private final CloseableHttpClient httpClient;
private File omSnapshotDBParentDir = null;
private String omDBSnapshotUrl;
@@ -104,6 +102,7 @@
private ReconTaskController reconTaskController;
private ReconTaskStatusDao reconTaskStatusDao;
private ReconUtils reconUtils;
+ private OzoneManagerSyncMetrics metrics;
/**
* OM Snapshot related task names.
@@ -121,6 +120,17 @@
ReconUtils reconUtils,
OzoneManagerProtocol ozoneManagerClient) {
+ int connectionTimeout = (int) configuration.getTimeDuration(
+ RECON_OM_CONNECTION_TIMEOUT,
+ RECON_OM_CONNECTION_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS);
+ int connectionRequestTimeout = (int)configuration.getTimeDuration(
+ RECON_OM_CONNECTION_REQUEST_TIMEOUT,
+ RECON_OM_CONNECTION_REQUEST_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS);
+
+ connectionFactory =
+ URLConnectionFactory.newDefaultURLConnectionFactory(connectionTimeout,
+ connectionRequestTimeout, configuration);
+
String ozoneManagerHttpAddress = configuration.get(OMConfigKeys
.OZONE_OM_HTTP_ADDRESS_KEY);
@@ -132,26 +142,6 @@
HttpConfig.Policy policy = HttpConfig.getHttpPolicy(configuration);
- int socketTimeout = (int) configuration.getTimeDuration(
- RECON_OM_SOCKET_TIMEOUT, RECON_OM_SOCKET_TIMEOUT_DEFAULT,
- TimeUnit.MILLISECONDS);
- int connectionTimeout = (int) configuration.getTimeDuration(
- RECON_OM_CONNECTION_TIMEOUT,
- RECON_OM_CONNECTION_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS);
- int connectionRequestTimeout = (int)configuration.getTimeDuration(
- RECON_OM_CONNECTION_REQUEST_TIMEOUT,
- RECON_OM_CONNECTION_REQUEST_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS);
-
- RequestConfig config = RequestConfig.custom()
- .setConnectTimeout(socketTimeout)
- .setConnectionRequestTimeout(connectionTimeout)
- .setSocketTimeout(connectionRequestTimeout).build();
-
- httpClient = HttpClientBuilder
- .create()
- .setDefaultRequestConfig(config)
- .build();
-
omDBSnapshotUrl = "http://" + ozoneManagerHttpAddress +
OZONE_OM_DB_CHECKPOINT_HTTP_ENDPOINT;
@@ -173,6 +163,7 @@
this.reconTaskStatusDao = reconTaskController.getReconTaskStatusDao();
this.ozoneManagerClient = ozoneManagerClient;
this.configuration = configuration;
+ this.metrics = OzoneManagerSyncMetrics.create();
}
public void registerOMDBTasks() {
@@ -239,6 +230,8 @@
reconTaskController.stop();
omMetadataManager.stop();
scheduler.shutdownNow();
+ metrics.unRegister();
+ connectionFactory.destroy();
}
/**
@@ -269,6 +262,11 @@
return omLeaderUrl;
}
+ private boolean isOmSpengoEnabled() {
+ return configuration.get(OZONE_OM_HTTP_AUTH_TYPE, "simple")
+ .equals("kerberos");
+ }
+
/**
* Method to obtain current OM DB Snapshot.
* @return DBCheckpoint instance.
@@ -280,11 +278,14 @@
File targetFile = new File(omSnapshotDBParentDir, snapshotFileName +
".tar.gz");
try {
- try (InputStream inputStream = reconUtils.makeHttpCall(httpClient,
- getOzoneManagerSnapshotUrl())) {
- FileUtils.copyInputStreamToFile(inputStream, targetFile);
- }
-
+ SecurityUtil.doAsLoginUser(() -> {
+ try (InputStream inputStream = reconUtils.makeHttpCall(
+ connectionFactory, getOzoneManagerSnapshotUrl(),
+ isOmSpengoEnabled())) {
+ FileUtils.copyInputStreamToFile(inputStream, targetFile);
+ }
+ return null;
+ });
// Untar the checkpoint file.
Path untarredDbDir = Paths.get(omSnapshotDBParentDir.getAbsolutePath(),
snapshotFileName);
@@ -308,7 +309,9 @@
boolean updateReconOmDBWithNewSnapshot() throws IOException {
// Obtain the current DB snapshot from OM and
// update the in house OM metadata managed DB instance.
+ long startTime = Time.monotonicNowNanos();
DBCheckpoint dbSnapshot = getOzoneManagerDBSnapshot();
+ metrics.updateSnapshotRequestLatency(Time.monotonicNowNanos() - startTime);
if (dbSnapshot != null && dbSnapshot.getCheckpointLocation() != null) {
LOG.info("Got new checkpoint from OM : " +
dbSnapshot.getCheckpointLocation());
@@ -339,13 +342,15 @@
throws IOException, RocksDBException {
DBUpdatesRequest dbUpdatesRequest = DBUpdatesRequest.newBuilder()
.setSequenceNumber(fromSequenceNumber).build();
- DBUpdatesWrapper dbUpdates = ozoneManagerClient.getDBUpdates(
- dbUpdatesRequest);
+ DBUpdates dbUpdates = ozoneManagerClient.getDBUpdates(dbUpdatesRequest);
if (null != dbUpdates) {
RDBStore rocksDBStore = (RDBStore) omMetadataManager.getStore();
RocksDB rocksDB = rocksDBStore.getDb();
- LOG.debug("Number of updates received from OM : {}",
- dbUpdates.getData().size());
+ int numUpdates = dbUpdates.getData().size();
+ LOG.info("Number of updates received from OM : {}", numUpdates);
+ if (numUpdates > 0) {
+ metrics.incrNumUpdatesInDeltaTotal(numUpdates);
+ }
for (byte[] data : dbUpdates.getData()) {
try (WriteBatch writeBatch = new WriteBatch(data)) {
writeBatch.iterate(omdbUpdatesHandler);
@@ -392,6 +397,7 @@
} catch (InterruptedException intEx) {
Thread.currentThread().interrupt();
} catch (Exception e) {
+ metrics.incrNumDeltaRequestsFailed();
LOG.warn("Unable to get and apply delta updates from OM.", e);
fullSnapshot = true;
}
@@ -399,6 +405,7 @@
if (fullSnapshot) {
try {
+ metrics.incrNumSnapshotRequests();
LOG.info("Obtaining full snapshot from Ozone Manager");
// Update local Recon OM DB to new snapshot.
boolean success = updateReconOmDBWithNewSnapshot();
@@ -417,6 +424,7 @@
} catch (InterruptedException intEx) {
Thread.currentThread().interrupt();
} catch (Exception e) {
+ metrics.incrNumSnapshotRequestsFailed();
LOG.error("Unable to update Recon's metadata with new OM DB. ", e);
}
}
@@ -429,5 +437,9 @@
private long getCurrentOMDBSequenceNumber() {
return omMetadataManager.getLastSequenceNumberFromDB();
}
+
+ public OzoneManagerSyncMetrics getMetrics() {
+ return metrics;
+ }
}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ReconTaskConfig.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ReconTaskConfig.java
new file mode 100644
index 0000000..688e3ac
--- /dev/null
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ReconTaskConfig.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.tasks;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.hdds.conf.Config;
+import org.apache.hadoop.hdds.conf.ConfigGroup;
+import org.apache.hadoop.hdds.conf.ConfigTag;
+import org.apache.hadoop.hdds.conf.ConfigType;
+
+/**
+ * The configuration class for the Recon tasks.
+ */
+@ConfigGroup(prefix = "ozone.recon.task")
+public class ReconTaskConfig {
+
+ @Config(key = "pipelinesync.interval",
+ type = ConfigType.TIME, timeUnit = TimeUnit.SECONDS,
+ defaultValue = "600s",
+ tags = { ConfigTag.RECON, ConfigTag.OZONE },
+ description = "The time interval of periodic sync of pipeline state " +
+ "from SCM to Recon."
+ )
+ private long pipelineSyncTaskInterval;
+
+ public long getPipelineSyncTaskInterval() {
+ return pipelineSyncTaskInterval;
+ }
+
+ public void setPipelineSyncTaskInterval(long pipelineSyncTaskInterval) {
+ this.pipelineSyncTaskInterval = pipelineSyncTaskInterval;
+ }
+
+ @Config(key = "missingcontainer.interval",
+ type = ConfigType.TIME, timeUnit = TimeUnit.SECONDS,
+ defaultValue = "300s",
+ tags = { ConfigTag.RECON, ConfigTag.OZONE },
+ description = "The time interval of the periodic check for " +
+ "containers with zero replicas in the cluster as reported by " +
+ "Datanodes."
+ )
+ private long missingContainerTaskInterval;
+
+ public long getMissingContainerTaskInterval() {
+ return missingContainerTaskInterval;
+ }
+
+ public void setMissingContainerTaskInterval(long interval) {
+ this.missingContainerTaskInterval = interval;
+ }
+
+}
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/LICENSE b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/LICENSE
index 31b74fa..3a21949 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/LICENSE
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/LICENSE
@@ -53,7 +53,7 @@
-----
-The following software may be included in this product: @babel/code-frame, @babel/core, @babel/generator, @babel/helper-annotate-as-pure, @babel/helper-builder-binary-assignment-operator-visitor, @babel/helper-builder-react-jsx, @babel/helper-call-delegate, @babel/helper-create-class-features-plugin, @babel/helper-create-regexp-features-plugin, @babel/helper-define-map, @babel/helper-explode-assignable-expression, @babel/helper-function-name, @babel/helper-get-function-arity, @babel/helper-hoist-variables, @babel/helper-member-expression-to-functions, @babel/helper-module-imports, @babel/helper-module-transforms, @babel/helper-optimise-call-expression, @babel/helper-regex, @babel/helper-remap-async-to-generator, @babel/helper-replace-supers, @babel/helper-simple-access, @babel/helper-split-export-declaration, @babel/helper-wrap-function, @babel/helpers, @babel/highlight, @babel/plugin-proposal-async-generator-functions, @babel/plugin-proposal-class-properties, @babel/plugin-proposal-decorators, @babel/plugin-proposal-dynamic-import, @babel/plugin-proposal-json-strings, @babel/plugin-proposal-object-rest-spread, @babel/plugin-proposal-optional-catch-binding, @babel/plugin-proposal-unicode-property-regex, @babel/plugin-syntax-async-generators, @babel/plugin-syntax-decorators, @babel/plugin-syntax-dynamic-import, @babel/plugin-syntax-flow, @babel/plugin-syntax-json-strings, @babel/plugin-syntax-jsx, @babel/plugin-syntax-object-rest-spread, @babel/plugin-syntax-optional-catch-binding, @babel/plugin-syntax-top-level-await, @babel/plugin-syntax-typescript, @babel/plugin-transform-arrow-functions, @babel/plugin-transform-async-to-generator, @babel/plugin-transform-block-scoped-functions, @babel/plugin-transform-block-scoping, @babel/plugin-transform-classes, @babel/plugin-transform-computed-properties, @babel/plugin-transform-destructuring, @babel/plugin-transform-dotall-regex, @babel/plugin-transform-duplicate-keys, @babel/plugin-transform-exponentiation-operator, @babel/plugin-transform-flow-strip-types, @babel/plugin-transform-for-of, @babel/plugin-transform-function-name, @babel/plugin-transform-literals, @babel/plugin-transform-member-expression-literals, @babel/plugin-transform-modules-amd, @babel/plugin-transform-modules-commonjs, @babel/plugin-transform-modules-systemjs, @babel/plugin-transform-modules-umd, @babel/plugin-transform-named-capturing-groups-regex, @babel/plugin-transform-new-target, @babel/plugin-transform-object-super, @babel/plugin-transform-parameters, @babel/plugin-transform-property-literals, @babel/plugin-transform-react-constant-elements, @babel/plugin-transform-react-display-name, @babel/plugin-transform-react-jsx, @babel/plugin-transform-react-jsx-self, @babel/plugin-transform-react-jsx-source, @babel/plugin-transform-regenerator, @babel/plugin-transform-reserved-words, @babel/plugin-transform-runtime, @babel/plugin-transform-shorthand-properties, @babel/plugin-transform-spread, @babel/plugin-transform-sticky-regex, @babel/plugin-transform-template-literals, @babel/plugin-transform-typeof-symbol, @babel/plugin-transform-typescript, @babel/plugin-transform-unicode-regex, @babel/preset-env, @babel/preset-react, @babel/preset-typescript, @babel/runtime, @babel/runtime-corejs3, @babel/template, @babel/traverse, @babel/types. A copy of the source code may be downloaded from https://github.com/babel/babel/tree/master/packages/babel-code-frame (@babel/code-frame), https://github.com/babel/babel/tree/master/packages/babel-core (@babel/core), https://github.com/babel/babel/tree/master/packages/babel-generator (@babel/generator), https://github.com/babel/babel/tree/master/packages/babel-helper-annotate-as-pure (@babel/helper-annotate-as-pure), https://github.com/babel/babel/tree/master/packages/babel-helper-builder-binary-assignment-operator-visitor (@babel/helper-builder-binary-assignment-operator-visitor), https://github.com/babel/babel/tree/master/packages/babel-helper-builder-react-jsx (@babel/helper-builder-react-jsx), https://github.com/babel/babel/tree/master/packages/babel-helper-call-delegate (@babel/helper-call-delegate), https://github.com/babel/babel/tree/master/packages/babel-helper-create-class-features-plugin (@babel/helper-create-class-features-plugin), https://github.com/babel/babel (@babel/helper-create-regexp-features-plugin), https://github.com/babel/babel/tree/master/packages/babel-helper-define-map (@babel/helper-define-map), https://github.com/babel/babel/tree/master/packages/babel-helper-explode-assignable-expression (@babel/helper-explode-assignable-expression), https://github.com/babel/babel/tree/master/packages/babel-helper-function-name (@babel/helper-function-name), https://github.com/babel/babel/tree/master/packages/babel-helper-get-function-arity (@babel/helper-get-function-arity), https://github.com/babel/babel/tree/master/packages/babel-helper-hoist-variables (@babel/helper-hoist-variables), https://github.com/babel/babel/tree/master/packages/babel-helper-member-expression-to-functions (@babel/helper-member-expression-to-functions), https://github.com/babel/babel/tree/master/packages/babel-helper-module-imports (@babel/helper-module-imports), https://github.com/babel/babel/tree/master/packages/babel-helper-module-transforms (@babel/helper-module-transforms), https://github.com/babel/babel/tree/master/packages/babel-helper-optimise-call-expression (@babel/helper-optimise-call-expression), https://github.com/babel/babel/tree/master/packages/babel-helper-regex (@babel/helper-regex), https://github.com/babel/babel/tree/master/packages/babel-helper-remap-async-to-generator (@babel/helper-remap-async-to-generator), https://github.com/babel/babel/tree/master/packages/babel-helper-replace-supers (@babel/helper-replace-supers), https://github.com/babel/babel/tree/master/packages/babel-helper-simple-access (@babel/helper-simple-access), https://github.com/babel/babel/tree/master/packages/babel-helper-split-export-declaration (@babel/helper-split-export-declaration), https://github.com/babel/babel/tree/master/packages/babel-helper-wrap-function (@babel/helper-wrap-function), https://github.com/babel/babel/tree/master/packages/babel-helpers (@babel/helpers), https://github.com/babel/babel/tree/master/packages/babel-highlight (@babel/highlight), https://github.com/babel/babel/tree/master/packages/babel-plugin-proposal-async-generator-functions (@babel/plugin-proposal-async-generator-functions), https://github.com/babel/babel/tree/master/packages/babel-plugin-proposal-class-properties (@babel/plugin-proposal-class-properties), https://github.com/babel/babel/tree/master/packages/babel-plugin-proposal-decorators (@babel/plugin-proposal-decorators), https://github.com/babel/babel/tree/master/packages/babel-plugin-proposal-dynamic-import (@babel/plugin-proposal-dynamic-import), https://github.com/babel/babel/tree/master/packages/babel-plugin-proposal-json-strings (@babel/plugin-proposal-json-strings), https://github.com/babel/babel/tree/master/packages/babel-plugin-proposal-object-rest-spread (@babel/plugin-proposal-object-rest-spread), https://github.com/babel/babel/tree/master/packages/babel-plugin-proposal-optional-catch-binding (@babel/plugin-proposal-optional-catch-binding), https://github.com/babel/babel/tree/master/packages/babel-plugin-proposal-unicode-property-regex (@babel/plugin-proposal-unicode-property-regex), https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-async-generators (@babel/plugin-syntax-async-generators), https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-decorators (@babel/plugin-syntax-decorators), https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-dynamic-import (@babel/plugin-syntax-dynamic-import), https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-flow (@babel/plugin-syntax-flow), https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-json-strings (@babel/plugin-syntax-json-strings), https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-jsx (@babel/plugin-syntax-jsx), https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-object-rest-spread (@babel/plugin-syntax-object-rest-spread), https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-optional-catch-binding (@babel/plugin-syntax-optional-catch-binding), https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-top-level-await (@babel/plugin-syntax-top-level-await), https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-typescript (@babel/plugin-syntax-typescript), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-arrow-functions (@babel/plugin-transform-arrow-functions), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-async-to-generator (@babel/plugin-transform-async-to-generator), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-block-scoped-functions (@babel/plugin-transform-block-scoped-functions), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-block-scoping (@babel/plugin-transform-block-scoping), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-classes (@babel/plugin-transform-classes), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-computed-properties (@babel/plugin-transform-computed-properties), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-destructuring (@babel/plugin-transform-destructuring), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-dotall-regex (@babel/plugin-transform-dotall-regex), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-duplicate-keys (@babel/plugin-transform-duplicate-keys), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-exponentiation-operator (@babel/plugin-transform-exponentiation-operator), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-flow-strip-types (@babel/plugin-transform-flow-strip-types), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-for-of (@babel/plugin-transform-for-of), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-function-name (@babel/plugin-transform-function-name), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-literals (@babel/plugin-transform-literals), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-member-expression-literals (@babel/plugin-transform-member-expression-literals), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-modules-amd (@babel/plugin-transform-modules-amd), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-modules-commonjs (@babel/plugin-transform-modules-commonjs), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-modules-systemjs (@babel/plugin-transform-modules-systemjs), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-modules-umd (@babel/plugin-transform-modules-umd), https://github.com/babel/babel.git (@babel/plugin-transform-named-capturing-groups-regex), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-new-target (@babel/plugin-transform-new-target), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-object-super (@babel/plugin-transform-object-super), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-parameters (@babel/plugin-transform-parameters), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-property-literals (@babel/plugin-transform-property-literals), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-react-constant-elements (@babel/plugin-transform-react-constant-elements), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-react-display-name (@babel/plugin-transform-react-display-name), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-react-jsx (@babel/plugin-transform-react-jsx), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-react-jsx-self (@babel/plugin-transform-react-jsx-self), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-react-jsx-source (@babel/plugin-transform-react-jsx-source), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-regenerator (@babel/plugin-transform-regenerator), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-reserved-words (@babel/plugin-transform-reserved-words), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-runtime (@babel/plugin-transform-runtime), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-shorthand-properties (@babel/plugin-transform-shorthand-properties), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-spread (@babel/plugin-transform-spread), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-sticky-regex (@babel/plugin-transform-sticky-regex), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-template-literals (@babel/plugin-transform-template-literals), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-typeof-symbol (@babel/plugin-transform-typeof-symbol), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-typescript (@babel/plugin-transform-typescript), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-unicode-regex (@babel/plugin-transform-unicode-regex), https://github.com/babel/babel/tree/master/packages/babel-preset-env (@babel/preset-env), https://github.com/babel/babel/tree/master/packages/babel-preset-react (@babel/preset-react), https://github.com/babel/babel/tree/master/packages/babel-preset-typescript (@babel/preset-typescript), https://github.com/babel/babel/tree/master/packages/babel-runtime (@babel/runtime), https://github.com/babel/babel/tree/master/packages/babel-runtime-corejs3 (@babel/runtime-corejs3), https://github.com/babel/babel/tree/master/packages/babel-template (@babel/template), https://github.com/babel/babel/tree/master/packages/babel-traverse (@babel/traverse), https://github.com/babel/babel/tree/master/packages/babel-types (@babel/types). This software contains the following license and notice below:
+The following software may be included in this product: @babel/code-frame, @babel/compat-data, @babel/core, @babel/generator, @babel/helper-annotate-as-pure, @babel/helper-builder-binary-assignment-operator-visitor, @babel/helper-builder-react-jsx, @babel/helper-builder-react-jsx-experimental, @babel/helper-compilation-targets, @babel/helper-create-class-features-plugin, @babel/helper-create-regexp-features-plugin, @babel/helper-define-map, @babel/helper-explode-assignable-expression, @babel/helper-function-name, @babel/helper-get-function-arity, @babel/helper-hoist-variables, @babel/helper-member-expression-to-functions, @babel/helper-module-imports, @babel/helper-module-transforms, @babel/helper-optimise-call-expression, @babel/helper-plugin-utils, @babel/helper-regex, @babel/helper-remap-async-to-generator, @babel/helper-replace-supers, @babel/helper-simple-access, @babel/helper-split-export-declaration, @babel/helper-validator-identifier, @babel/helper-wrap-function, @babel/helpers, @babel/highlight, @babel/plugin-proposal-async-generator-functions, @babel/plugin-proposal-class-properties, @babel/plugin-proposal-decorators, @babel/plugin-proposal-dynamic-import, @babel/plugin-proposal-json-strings, @babel/plugin-proposal-nullish-coalescing-operator, @babel/plugin-proposal-numeric-separator, @babel/plugin-proposal-object-rest-spread, @babel/plugin-proposal-optional-catch-binding, @babel/plugin-proposal-optional-chaining, @babel/plugin-proposal-unicode-property-regex, @babel/plugin-syntax-async-generators, @babel/plugin-syntax-decorators, @babel/plugin-syntax-dynamic-import, @babel/plugin-syntax-flow, @babel/plugin-syntax-json-strings, @babel/plugin-syntax-jsx, @babel/plugin-syntax-nullish-coalescing-operator, @babel/plugin-syntax-numeric-separator, @babel/plugin-syntax-object-rest-spread, @babel/plugin-syntax-optional-catch-binding, @babel/plugin-syntax-optional-chaining, @babel/plugin-syntax-top-level-await, @babel/plugin-syntax-typescript, @babel/plugin-transform-arrow-functions, @babel/plugin-transform-async-to-generator, @babel/plugin-transform-block-scoped-functions, @babel/plugin-transform-block-scoping, @babel/plugin-transform-classes, @babel/plugin-transform-computed-properties, @babel/plugin-transform-destructuring, @babel/plugin-transform-dotall-regex, @babel/plugin-transform-duplicate-keys, @babel/plugin-transform-exponentiation-operator, @babel/plugin-transform-flow-strip-types, @babel/plugin-transform-for-of, @babel/plugin-transform-function-name, @babel/plugin-transform-literals, @babel/plugin-transform-member-expression-literals, @babel/plugin-transform-modules-amd, @babel/plugin-transform-modules-commonjs, @babel/plugin-transform-modules-systemjs, @babel/plugin-transform-modules-umd, @babel/plugin-transform-named-capturing-groups-regex, @babel/plugin-transform-new-target, @babel/plugin-transform-object-super, @babel/plugin-transform-parameters, @babel/plugin-transform-property-literals, @babel/plugin-transform-react-constant-elements, @babel/plugin-transform-react-display-name, @babel/plugin-transform-react-jsx, @babel/plugin-transform-react-jsx-development, @babel/plugin-transform-react-jsx-self, @babel/plugin-transform-react-jsx-source, @babel/plugin-transform-regenerator, @babel/plugin-transform-reserved-words, @babel/plugin-transform-runtime, @babel/plugin-transform-shorthand-properties, @babel/plugin-transform-spread, @babel/plugin-transform-sticky-regex, @babel/plugin-transform-template-literals, @babel/plugin-transform-typeof-symbol, @babel/plugin-transform-typescript, @babel/plugin-transform-unicode-regex, @babel/preset-env, @babel/preset-react, @babel/preset-typescript, @babel/runtime, @babel/runtime-corejs3, @babel/template, @babel/traverse, @babel/types. A copy of the source code may be downloaded from https://github.com/babel/babel/tree/master/packages/babel-code-frame (@babel/code-frame), https://github.com/babel/babel/tree/master/packages/babel-compat-data (@babel/compat-data), https://github.com/babel/babel/tree/master/packages/babel-core (@babel/core), https://github.com/babel/babel/tree/master/packages/babel-generator (@babel/generator), https://github.com/babel/babel/tree/master/packages/babel-helper-annotate-as-pure (@babel/helper-annotate-as-pure), https://github.com/babel/babel/tree/master/packages/babel-helper-builder-binary-assignment-operator-visitor (@babel/helper-builder-binary-assignment-operator-visitor), https://github.com/babel/babel/tree/master/packages/babel-helper-builder-react-jsx (@babel/helper-builder-react-jsx), https://github.com/babel/babel/tree/master/packages/babel-helper-builder-react-jsx-experimental (@babel/helper-builder-react-jsx-experimental), https://github.com/babel/babel/tree/master/packages/babel-helper-compilation-targets (@babel/helper-compilation-targets), https://github.com/babel/babel/tree/master/packages/babel-helper-create-class-features-plugin (@babel/helper-create-class-features-plugin), https://github.com/babel/babel (@babel/helper-create-regexp-features-plugin), https://github.com/babel/babel/tree/master/packages/babel-helper-define-map (@babel/helper-define-map), https://github.com/babel/babel/tree/master/packages/babel-helper-explode-assignable-expression (@babel/helper-explode-assignable-expression), https://github.com/babel/babel/tree/master/packages/babel-helper-function-name (@babel/helper-function-name), https://github.com/babel/babel/tree/master/packages/babel-helper-get-function-arity (@babel/helper-get-function-arity), https://github.com/babel/babel/tree/master/packages/babel-helper-hoist-variables (@babel/helper-hoist-variables), https://github.com/babel/babel/tree/master/packages/babel-helper-member-expression-to-functions (@babel/helper-member-expression-to-functions), https://github.com/babel/babel/tree/master/packages/babel-helper-module-imports (@babel/helper-module-imports), https://github.com/babel/babel/tree/master/packages/babel-helper-module-transforms (@babel/helper-module-transforms), https://github.com/babel/babel/tree/master/packages/babel-helper-optimise-call-expression (@babel/helper-optimise-call-expression), https://github.com/babel/babel/tree/master/packages/babel-helper-plugin-utils (@babel/helper-plugin-utils), https://github.com/babel/babel/tree/master/packages/babel-helper-regex (@babel/helper-regex), https://github.com/babel/babel/tree/master/packages/babel-helper-remap-async-to-generator (@babel/helper-remap-async-to-generator), https://github.com/babel/babel/tree/master/packages/babel-helper-replace-supers (@babel/helper-replace-supers), https://github.com/babel/babel/tree/master/packages/babel-helper-simple-access (@babel/helper-simple-access), https://github.com/babel/babel/tree/master/packages/babel-helper-split-export-declaration (@babel/helper-split-export-declaration), https://github.com/babel/babel/tree/master/packages/babel-helper-validator-identifier (@babel/helper-validator-identifier), https://github.com/babel/babel/tree/master/packages/babel-helper-wrap-function (@babel/helper-wrap-function), https://github.com/babel/babel/tree/master/packages/babel-helpers (@babel/helpers), https://github.com/babel/babel/tree/master/packages/babel-highlight (@babel/highlight), https://github.com/babel/babel/tree/master/packages/babel-plugin-proposal-async-generator-functions (@babel/plugin-proposal-async-generator-functions), https://github.com/babel/babel/tree/master/packages/babel-plugin-proposal-class-properties (@babel/plugin-proposal-class-properties), https://github.com/babel/babel/tree/master/packages/babel-plugin-proposal-decorators (@babel/plugin-proposal-decorators), https://github.com/babel/babel/tree/master/packages/babel-plugin-proposal-dynamic-import (@babel/plugin-proposal-dynamic-import), https://github.com/babel/babel/tree/master/packages/babel-plugin-proposal-json-strings (@babel/plugin-proposal-json-strings), https://github.com/babel/babel/tree/master/packages/babel-plugin-proposal-nullish-coalescing-operator (@babel/plugin-proposal-nullish-coalescing-operator), https://github.com/babel/babel/tree/master/packages/babel-plugin-proposal-numeric-separator (@babel/plugin-proposal-numeric-separator), https://github.com/babel/babel/tree/master/packages/babel-plugin-proposal-object-rest-spread (@babel/plugin-proposal-object-rest-spread), https://github.com/babel/babel/tree/master/packages/babel-plugin-proposal-optional-catch-binding (@babel/plugin-proposal-optional-catch-binding), https://github.com/babel/babel/tree/master/packages/babel-plugin-proposal-optional-chaining (@babel/plugin-proposal-optional-chaining), https://github.com/babel/babel/tree/master/packages/babel-plugin-proposal-unicode-property-regex (@babel/plugin-proposal-unicode-property-regex), https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-async-generators (@babel/plugin-syntax-async-generators), https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-decorators (@babel/plugin-syntax-decorators), https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-dynamic-import (@babel/plugin-syntax-dynamic-import), https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-flow (@babel/plugin-syntax-flow), https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-json-strings (@babel/plugin-syntax-json-strings), https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-jsx (@babel/plugin-syntax-jsx), https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-nullish-coalescing-operator (@babel/plugin-syntax-nullish-coalescing-operator), https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-numeric-separator (@babel/plugin-syntax-numeric-separator), https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-object-rest-spread (@babel/plugin-syntax-object-rest-spread), https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-optional-catch-binding (@babel/plugin-syntax-optional-catch-binding), https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-optional-chaining (@babel/plugin-syntax-optional-chaining), https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-top-level-await (@babel/plugin-syntax-top-level-await), https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-typescript (@babel/plugin-syntax-typescript), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-arrow-functions (@babel/plugin-transform-arrow-functions), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-async-to-generator (@babel/plugin-transform-async-to-generator), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-block-scoped-functions (@babel/plugin-transform-block-scoped-functions), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-block-scoping (@babel/plugin-transform-block-scoping), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-classes (@babel/plugin-transform-classes), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-computed-properties (@babel/plugin-transform-computed-properties), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-destructuring (@babel/plugin-transform-destructuring), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-dotall-regex (@babel/plugin-transform-dotall-regex), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-duplicate-keys (@babel/plugin-transform-duplicate-keys), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-exponentiation-operator (@babel/plugin-transform-exponentiation-operator), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-flow-strip-types (@babel/plugin-transform-flow-strip-types), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-for-of (@babel/plugin-transform-for-of), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-function-name (@babel/plugin-transform-function-name), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-literals (@babel/plugin-transform-literals), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-member-expression-literals (@babel/plugin-transform-member-expression-literals), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-modules-amd (@babel/plugin-transform-modules-amd), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-modules-commonjs (@babel/plugin-transform-modules-commonjs), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-modules-systemjs (@babel/plugin-transform-modules-systemjs), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-modules-umd (@babel/plugin-transform-modules-umd), https://github.com/babel/babel.git (@babel/plugin-transform-named-capturing-groups-regex), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-new-target (@babel/plugin-transform-new-target), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-object-super (@babel/plugin-transform-object-super), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-parameters (@babel/plugin-transform-parameters), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-property-literals (@babel/plugin-transform-property-literals), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-react-constant-elements (@babel/plugin-transform-react-constant-elements), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-react-display-name (@babel/plugin-transform-react-display-name), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-react-jsx (@babel/plugin-transform-react-jsx), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-react-jsx-development (@babel/plugin-transform-react-jsx-development), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-react-jsx-self (@babel/plugin-transform-react-jsx-self), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-react-jsx-source (@babel/plugin-transform-react-jsx-source), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-regenerator (@babel/plugin-transform-regenerator), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-reserved-words (@babel/plugin-transform-reserved-words), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-runtime (@babel/plugin-transform-runtime), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-shorthand-properties (@babel/plugin-transform-shorthand-properties), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-spread (@babel/plugin-transform-spread), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-sticky-regex (@babel/plugin-transform-sticky-regex), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-template-literals (@babel/plugin-transform-template-literals), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-typeof-symbol (@babel/plugin-transform-typeof-symbol), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-typescript (@babel/plugin-transform-typescript), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-unicode-regex (@babel/plugin-transform-unicode-regex), https://github.com/babel/babel/tree/master/packages/babel-preset-env (@babel/preset-env), https://github.com/babel/babel/tree/master/packages/babel-preset-react (@babel/preset-react), https://github.com/babel/babel/tree/master/packages/babel-preset-typescript (@babel/preset-typescript), https://github.com/babel/babel.git (@babel/runtime), https://github.com/babel/babel/tree/master/packages/babel-runtime-corejs3 (@babel/runtime-corejs3), https://github.com/babel/babel/tree/master/packages/babel-template (@babel/template), https://github.com/babel/babel/tree/master/packages/babel-traverse (@babel/traverse), https://github.com/babel/babel/tree/master/packages/babel-types (@babel/types). This software contains the following license and notice below:
MIT License
@@ -80,7 +80,7 @@
-----
-The following software may be included in this product: @babel/helper-plugin-utils, @babel/preset-react. A copy of the source code may be downloaded from https://github.com/babel/babel/tree/master/packages/babel-helper-plugin-utils (@babel/helper-plugin-utils), https://github.com/babel/babel/tree/master/packages/babel-preset-react (@babel/preset-react). This software contains the following license and notice below:
+The following software may be included in this product: @babel/helper-plugin-utils. A copy of the source code may be downloaded from https://github.com/babel/babel/tree/master/packages/babel-helper-plugin-utils. This software contains the following license and notice below:
MIT License
@@ -131,33 +131,6 @@
-----
-The following software may be included in this product: @babel/plugin-syntax-dynamic-import, @babel/plugin-transform-react-constant-elements, @babel/plugin-transform-react-display-name. A copy of the source code may be downloaded from https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-dynamic-import (@babel/plugin-syntax-dynamic-import), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-react-constant-elements (@babel/plugin-transform-react-constant-elements), https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-react-display-name (@babel/plugin-transform-react-display-name). This software contains the following license and notice below:
-
-MIT License
-
-Copyright (c) 2014-2018 Sebastian McKenzie and other contributors
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
------
-
The following software may be included in this product: @cnakazawa/watch, aws-sign2, forever-agent, oauth-sign, request, tunnel-agent. A copy of the source code may be downloaded from git://github.com/mikeal/watch.git (@cnakazawa/watch), https://github.com/mikeal/aws-sign (aws-sign2), https://github.com/mikeal/forever-agent (forever-agent), https://github.com/mikeal/oauth-sign (oauth-sign), https://github.com/request/request.git (request), https://github.com/mikeal/tunnel-agent (tunnel-agent). This software contains the following license and notice below:
Apache License
@@ -218,7 +191,7 @@
-----
-The following software may be included in this product: @csstools/convert-colors, css-blank-pseudo, css-has-pseudo, css-prefers-color-scheme, postcss-browser-comments, postcss-color-functional-notation, postcss-color-mod-function, postcss-dir-pseudo-class, postcss-double-position-gradients, postcss-env-function, postcss-focus-visible, postcss-focus-within, postcss-gap-properties, postcss-image-set-function, postcss-lab-function, postcss-logical, postcss-nesting, postcss-normalize, postcss-overflow-shorthand, postcss-place, postcss-preset-env, postcss-pseudo-class-any-link. A copy of the source code may be downloaded from https://github.com/jonathantneal/convert-colors.git (@csstools/convert-colors), https://github.com/csstools/css-blank-pseudo.git (css-blank-pseudo), https://github.com/csstools/css-has-pseudo.git (css-has-pseudo), https://github.com/csstools/css-prefers-color-scheme.git (css-prefers-color-scheme), https://github.com/csstools/postcss-browser-comments.git (postcss-browser-comments), https://github.com/jonathantneal/postcss-color-functional-notation.git (postcss-color-functional-notation), https://github.com/jonathantneal/postcss-color-mod-function.git (postcss-color-mod-function), https://github.com/jonathantneal/postcss-dir-pseudo-class.git (postcss-dir-pseudo-class), https://github.com/jonathantneal/postcss-double-position-gradients.git (postcss-double-position-gradients), https://github.com/jonathantneal/postcss-env-function.git (postcss-env-function), https://github.com/jonathantneal/postcss-focus-visible.git (postcss-focus-visible), https://github.com/jonathantneal/postcss-focus-within.git (postcss-focus-within), https://github.com/jonathantneal/postcss-gap-properties.git (postcss-gap-properties), https://github.com/jonathantneal/postcss-image-set-function.git (postcss-image-set-function), https://github.com/jonathantneal/postcss-lab-function.git (postcss-lab-function), https://github.com/jonathantneal/postcss-logical.git (postcss-logical), https://github.com/jonathantneal/postcss-nesting.git (postcss-nesting), https://github.com/csstools/postcss-normalize.git (postcss-normalize), https://github.com/jonathantneal/postcss-overflow-shorthand.git (postcss-overflow-shorthand), https://github.com/jonathantneal/postcss-place.git (postcss-place), https://github.com/csstools/postcss-preset-env.git (postcss-preset-env), https://github.com/jonathantneal/postcss-pseudo-class-any-link.git (postcss-pseudo-class-any-link). This software contains the following license and notice below:
+The following software may be included in this product: @csstools/convert-colors, @csstools/normalize.css, css-blank-pseudo, css-has-pseudo, css-prefers-color-scheme, postcss-browser-comments, postcss-color-functional-notation, postcss-color-mod-function, postcss-dir-pseudo-class, postcss-double-position-gradients, postcss-env-function, postcss-focus-visible, postcss-focus-within, postcss-gap-properties, postcss-image-set-function, postcss-lab-function, postcss-logical, postcss-nesting, postcss-normalize, postcss-overflow-shorthand, postcss-place, postcss-preset-env, postcss-pseudo-class-any-link, sanitize.css. A copy of the source code may be downloaded from https://github.com/jonathantneal/convert-colors.git (@csstools/convert-colors), https://github.com/csstools/normalize.css.git (@csstools/normalize.css), https://github.com/csstools/css-blank-pseudo.git (css-blank-pseudo), https://github.com/csstools/css-has-pseudo.git (css-has-pseudo), https://github.com/csstools/css-prefers-color-scheme.git (css-prefers-color-scheme), https://github.com/csstools/postcss-browser-comments.git (postcss-browser-comments), https://github.com/jonathantneal/postcss-color-functional-notation.git (postcss-color-functional-notation), https://github.com/jonathantneal/postcss-color-mod-function.git (postcss-color-mod-function), https://github.com/jonathantneal/postcss-dir-pseudo-class.git (postcss-dir-pseudo-class), https://github.com/jonathantneal/postcss-double-position-gradients.git (postcss-double-position-gradients), https://github.com/jonathantneal/postcss-env-function.git (postcss-env-function), https://github.com/jonathantneal/postcss-focus-visible.git (postcss-focus-visible), https://github.com/jonathantneal/postcss-focus-within.git (postcss-focus-within), https://github.com/jonathantneal/postcss-gap-properties.git (postcss-gap-properties), https://github.com/jonathantneal/postcss-image-set-function.git (postcss-image-set-function), https://github.com/jonathantneal/postcss-lab-function.git (postcss-lab-function), https://github.com/jonathantneal/postcss-logical.git (postcss-logical), https://github.com/jonathantneal/postcss-nesting.git (postcss-nesting), https://github.com/csstools/postcss-normalize.git (postcss-normalize), https://github.com/jonathantneal/postcss-overflow-shorthand.git (postcss-overflow-shorthand), https://github.com/jonathantneal/postcss-place.git (postcss-place), https://github.com/csstools/postcss-preset-env.git (postcss-preset-env), https://github.com/jonathantneal/postcss-pseudo-class-any-link.git (postcss-pseudo-class-any-link), https://github.com/csstools/sanitize.css.git (sanitize.css). This software contains the following license and notice below:
# CC0 1.0 Universal
@@ -331,32 +304,6 @@
-----
-The following software may be included in this product: @csstools/normalize.css. A copy of the source code may be downloaded from https://github.com/csstools/normalize.css.git. This software contains the following license and notice below:
-
-# The MIT License (MIT)
-
-Copyright © Jonathan Neal and Nicolas Gallagher
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
------
-
The following software may be included in this product: @hapi/address. A copy of the source code may be downloaded from git://github.com/hapijs/address. This software contains the following license and notice below:
Copyright (c) 2019, Project contributors
@@ -471,7 +418,7 @@
-----
-The following software may be included in this product: @sindresorhus/is, ansi-escapes, ansi-regex, ansi-styles, binary-extensions, boxen, callsites, camelcase, chalk, cli-boxes, decamelize, del, execa, find-up, get-stream, global-dirs, globals, globby, got, gzip-size, has-flag, has-yarn, import-fresh, import-local, internal-ip, invert-kv, is-generator-fn, is-installed-globally, is-npm, is-root, is-svg, latest-version, lcid, leven, locate-path, lowercase-keys, make-dir, mem, mimic-fn, mimic-response, normalize-url, open, opn, os-locale, p-cancelable, p-is-promise, p-limit, p-locate, p-map, p-try, package-json, parent-module, parse-json, parse-ms, path-type, pify, pkg-dir, prepend-http, pretty-bytes, pretty-ms, read-pkg, read-pkg-up, registry-url, resolve-from, slash, string-length, string-width, strip-ansi, supports-color, to-readable-stream, type-fest, url-parse-lax, widest-line, wrap-ansi. A copy of the source code may be downloaded from https://github.com/sindresorhus/is.git (@sindresorhus/is), https://github.com/sindresorhus/ansi-escapes.git (ansi-escapes), https://github.com/chalk/ansi-regex.git (ansi-regex), https://github.com/chalk/ansi-styles.git (ansi-styles), https://github.com/sindresorhus/binary-extensions.git (binary-extensions), https://github.com/sindresorhus/boxen.git (boxen), https://github.com/sindresorhus/callsites.git (callsites), https://github.com/sindresorhus/camelcase.git (camelcase), https://github.com/chalk/chalk.git (chalk), https://github.com/sindresorhus/cli-boxes.git (cli-boxes), https://github.com/sindresorhus/decamelize.git (decamelize), https://github.com/sindresorhus/del.git (del), https://github.com/sindresorhus/execa.git (execa), https://github.com/sindresorhus/find-up.git (find-up), https://github.com/sindresorhus/get-stream.git (get-stream), https://github.com/sindresorhus/global-dirs.git (global-dirs), https://github.com/sindresorhus/globals.git (globals), https://github.com/sindresorhus/globby.git (globby), https://github.com/sindresorhus/got.git (got), https://github.com/sindresorhus/gzip-size.git (gzip-size), https://github.com/sindresorhus/has-flag.git (has-flag), https://github.com/sindresorhus/has-yarn.git (has-yarn), https://github.com/sindresorhus/import-fresh.git (import-fresh), https://github.com/sindresorhus/import-local.git (import-local), https://github.com/sindresorhus/internal-ip.git (internal-ip), https://github.com/sindresorhus/invert-kv.git (invert-kv), https://github.com/sindresorhus/is-generator-fn.git (is-generator-fn), https://github.com/sindresorhus/is-installed-globally.git (is-installed-globally), https://github.com/sindresorhus/is-npm.git (is-npm), https://github.com/sindresorhus/is-root.git (is-root), https://github.com/sindresorhus/is-svg.git (is-svg), https://github.com/sindresorhus/latest-version.git (latest-version), https://github.com/sindresorhus/lcid.git (lcid), https://github.com/sindresorhus/leven.git (leven), https://github.com/sindresorhus/locate-path.git (locate-path), https://github.com/sindresorhus/lowercase-keys.git (lowercase-keys), https://github.com/sindresorhus/make-dir.git (make-dir), https://github.com/sindresorhus/mem.git (mem), https://github.com/sindresorhus/mimic-fn.git (mimic-fn), https://github.com/sindresorhus/mimic-response.git (mimic-response), https://github.com/sindresorhus/normalize-url.git (normalize-url), https://github.com/sindresorhus/open.git (open), https://github.com/sindresorhus/opn.git (opn), https://github.com/sindresorhus/os-locale.git (os-locale), https://github.com/sindresorhus/p-cancelable.git (p-cancelable), https://github.com/sindresorhus/p-is-promise.git (p-is-promise), https://github.com/sindresorhus/p-limit.git (p-limit), https://github.com/sindresorhus/p-locate.git (p-locate), https://github.com/sindresorhus/p-map.git (p-map), https://github.com/sindresorhus/p-try.git (p-try), https://github.com/sindresorhus/package-json.git (package-json), https://github.com/sindresorhus/parent-module.git (parent-module), https://github.com/sindresorhus/parse-json.git (parse-json), https://github.com/sindresorhus/parse-ms.git (parse-ms), https://github.com/sindresorhus/path-type.git (path-type), https://github.com/sindresorhus/pify.git (pify), https://github.com/sindresorhus/pkg-dir.git (pkg-dir), https://github.com/sindresorhus/prepend-http.git (prepend-http), https://github.com/sindresorhus/pretty-bytes.git (pretty-bytes), https://github.com/sindresorhus/pretty-ms.git (pretty-ms), https://github.com/sindresorhus/read-pkg.git (read-pkg), https://github.com/sindresorhus/read-pkg-up.git (read-pkg-up), https://github.com/sindresorhus/registry-url.git (registry-url), https://github.com/sindresorhus/resolve-from.git (resolve-from), https://github.com/sindresorhus/slash.git (slash), https://github.com/sindresorhus/string-length.git (string-length), https://github.com/sindresorhus/string-width.git (string-width), https://github.com/chalk/strip-ansi.git (strip-ansi), https://github.com/chalk/supports-color.git (supports-color), https://github.com/sindresorhus/to-readable-stream.git (to-readable-stream), https://github.com/sindresorhus/type-fest.git (type-fest), https://github.com/sindresorhus/url-parse-lax.git (url-parse-lax), https://github.com/sindresorhus/widest-line.git (widest-line), https://github.com/chalk/wrap-ansi.git (wrap-ansi). This software contains the following license and notice below:
+The following software may be included in this product: @sindresorhus/is, aggregate-error, ansi-escapes, ansi-regex, ansi-styles, arrify, binary-extensions, boxen, callsites, camelcase, chalk, clean-stack, cli-boxes, cli-cursor, crypto-random-string, del, dot-prop, env-editor, escape-goat, escape-string-regexp, eslint-config-xo, eslint-config-xo-react, eslint-config-xo-typescript, eslint-formatter-pretty, eslint-plugin-ava, execa, find-up, get-stdin, get-stream, global-dirs, globals, globby, got, gzip-size, has-flag, has-yarn, import-fresh, import-local, import-modules, indent-string, internal-ip, invert-kv, irregular-plurals, is-absolute-url, is-docker, is-fullwidth-code-point, is-generator-fn, is-installed-globally, is-npm, is-obj, is-path-cwd, is-path-in-cwd, is-path-inside, is-root, is-svg, is-wsl, latest-version, lcid, leven, line-column-path, locate-path, log-symbols, lowercase-keys, make-dir, mem, meow, mimic-fn, mimic-response, normalize-url, onetime, open, open-editor, opn, os-locale, p-cancelable, p-is-promise, p-limit, p-locate, p-map, p-reduce, p-retry, p-try, package-json, parent-module, parse-json, parse-ms, path-exists, path-key, path-type, pify, pkg-dir, pkg-up, plur, prepend-http, pretty-bytes, pretty-ms, proto-props, pupa, quick-lru, read-pkg, read-pkg-up, registry-url, resolve-cwd, resolve-from, restore-cursor, semver-diff, shebang-regex, slash, string-length, string-width, strip-ansi, strip-json-comments, supports-color, term-size, to-readable-stream, type-fest, unique-string, url-parse-lax, widest-line, wrap-ansi, xdg-basedir. A copy of the source code may be downloaded from https://github.com/sindresorhus/is.git (@sindresorhus/is), https://github.com/sindresorhus/aggregate-error.git (aggregate-error), https://github.com/sindresorhus/ansi-escapes.git (ansi-escapes), https://github.com/chalk/ansi-regex.git (ansi-regex), https://github.com/chalk/ansi-styles.git (ansi-styles), https://github.com/sindresorhus/arrify.git (arrify), https://github.com/sindresorhus/binary-extensions.git (binary-extensions), https://github.com/sindresorhus/boxen.git (boxen), https://github.com/sindresorhus/callsites.git (callsites), https://github.com/sindresorhus/camelcase.git (camelcase), https://github.com/chalk/chalk.git (chalk), https://github.com/sindresorhus/clean-stack.git (clean-stack), https://github.com/sindresorhus/cli-boxes.git (cli-boxes), https://github.com/sindresorhus/cli-cursor.git (cli-cursor), https://github.com/sindresorhus/crypto-random-string.git (crypto-random-string), https://github.com/sindresorhus/del.git (del), https://github.com/sindresorhus/dot-prop.git (dot-prop), https://github.com/sindresorhus/env-editor.git (env-editor), https://github.com/sindresorhus/escape-goat.git (escape-goat), https://github.com/sindresorhus/escape-string-regexp.git (escape-string-regexp), https://github.com/xojs/eslint-config-xo.git (eslint-config-xo), https://github.com/xojs/eslint-config-xo-react.git (eslint-config-xo-react), https://github.com/xojs/eslint-config-xo-typescript.git (eslint-config-xo-typescript), https://github.com/sindresorhus/eslint-formatter-pretty.git (eslint-formatter-pretty), https://github.com/avajs/eslint-plugin-ava.git (eslint-plugin-ava), https://github.com/sindresorhus/execa.git (execa), https://github.com/sindresorhus/find-up.git (find-up), https://github.com/sindresorhus/get-stdin.git (get-stdin), https://github.com/sindresorhus/get-stream.git (get-stream), https://github.com/sindresorhus/global-dirs.git (global-dirs), https://github.com/sindresorhus/globals.git (globals), https://github.com/sindresorhus/globby.git (globby), https://github.com/sindresorhus/got.git (got), https://github.com/sindresorhus/gzip-size.git (gzip-size), https://github.com/sindresorhus/has-flag.git (has-flag), https://github.com/sindresorhus/has-yarn.git (has-yarn), https://github.com/sindresorhus/import-fresh.git (import-fresh), https://github.com/sindresorhus/import-local.git (import-local), https://github.com/sindresorhus/import-files.git (import-modules), https://github.com/sindresorhus/indent-string.git (indent-string), https://github.com/sindresorhus/internal-ip.git (internal-ip), https://github.com/sindresorhus/invert-kv.git (invert-kv), https://github.com/sindresorhus/irregular-plurals.git (irregular-plurals), https://github.com/sindresorhus/is-absolute-url.git (is-absolute-url), https://github.com/sindresorhus/is-docker.git (is-docker), https://github.com/sindresorhus/is-fullwidth-code-point.git (is-fullwidth-code-point), https://github.com/sindresorhus/is-generator-fn.git (is-generator-fn), https://github.com/sindresorhus/is-installed-globally.git (is-installed-globally), https://github.com/sindresorhus/is-npm.git (is-npm), https://github.com/sindresorhus/is-obj.git (is-obj), https://github.com/sindresorhus/is-path-cwd.git (is-path-cwd), https://github.com/sindresorhus/is-path-in-cwd.git (is-path-in-cwd), https://github.com/sindresorhus/is-path-inside.git (is-path-inside), https://github.com/sindresorhus/is-root.git (is-root), https://github.com/sindresorhus/is-svg.git (is-svg), https://github.com/sindresorhus/is-wsl.git (is-wsl), https://github.com/sindresorhus/latest-version.git (latest-version), https://github.com/sindresorhus/lcid.git (lcid), https://github.com/sindresorhus/leven.git (leven), https://github.com/sindresorhus/line-column-path.git (line-column-path), https://github.com/sindresorhus/locate-path.git (locate-path), https://github.com/sindresorhus/log-symbols.git (log-symbols), https://github.com/sindresorhus/lowercase-keys.git (lowercase-keys), https://github.com/sindresorhus/make-dir.git (make-dir), https://github.com/sindresorhus/mem.git (mem), https://github.com/sindresorhus/meow.git (meow), https://github.com/sindresorhus/mimic-fn.git (mimic-fn), https://github.com/sindresorhus/mimic-response.git (mimic-response), https://github.com/sindresorhus/normalize-url.git (normalize-url), https://github.com/sindresorhus/onetime.git (onetime), https://github.com/sindresorhus/open.git (open), https://github.com/sindresorhus/open-editor.git (open-editor), https://github.com/sindresorhus/opn.git (opn), https://github.com/sindresorhus/os-locale.git (os-locale), https://github.com/sindresorhus/p-cancelable.git (p-cancelable), https://github.com/sindresorhus/p-is-promise.git (p-is-promise), https://github.com/sindresorhus/p-limit.git (p-limit), https://github.com/sindresorhus/p-locate.git (p-locate), https://github.com/sindresorhus/p-map.git (p-map), https://github.com/sindresorhus/p-reduce.git (p-reduce), https://github.com/sindresorhus/p-retry.git (p-retry), https://github.com/sindresorhus/p-try.git (p-try), https://github.com/sindresorhus/package-json.git (package-json), https://github.com/sindresorhus/parent-module.git (parent-module), https://github.com/sindresorhus/parse-json.git (parse-json), https://github.com/sindresorhus/parse-ms.git (parse-ms), https://github.com/sindresorhus/path-exists.git (path-exists), https://github.com/sindresorhus/path-key.git (path-key), https://github.com/sindresorhus/path-type.git (path-type), https://github.com/sindresorhus/pify.git (pify), https://github.com/sindresorhus/pkg-dir.git (pkg-dir), https://github.com/sindresorhus/pkg-up.git (pkg-up), https://github.com/sindresorhus/plur.git (plur), https://github.com/sindresorhus/prepend-http.git (prepend-http), https://github.com/sindresorhus/pretty-bytes.git (pretty-bytes), https://github.com/sindresorhus/pretty-ms.git (pretty-ms), https://github.com/sindresorhus/proto-props.git (proto-props), https://github.com/sindresorhus/pupa.git (pupa), https://github.com/sindresorhus/quick-lru.git (quick-lru), https://github.com/sindresorhus/read-pkg.git (read-pkg), https://github.com/sindresorhus/read-pkg-up.git (read-pkg-up), https://github.com/sindresorhus/registry-url.git (registry-url), https://github.com/sindresorhus/resolve-cwd.git (resolve-cwd), https://github.com/sindresorhus/resolve-from.git (resolve-from), https://github.com/sindresorhus/restore-cursor.git (restore-cursor), https://github.com/sindresorhus/semver-diff.git (semver-diff), https://github.com/sindresorhus/shebang-regex.git (shebang-regex), https://github.com/sindresorhus/slash.git (slash), https://github.com/sindresorhus/string-length.git (string-length), https://github.com/sindresorhus/string-width.git (string-width), https://github.com/chalk/strip-ansi.git (strip-ansi), https://github.com/sindresorhus/strip-json-comments.git (strip-json-comments), https://github.com/chalk/supports-color.git (supports-color), https://github.com/sindresorhus/term-size.git (term-size), https://github.com/sindresorhus/to-readable-stream.git (to-readable-stream), https://github.com/sindresorhus/type-fest.git (type-fest), https://github.com/sindresorhus/unique-string.git (unique-string), https://github.com/sindresorhus/url-parse-lax.git (url-parse-lax), https://github.com/sindresorhus/widest-line.git (widest-line), https://github.com/chalk/wrap-ansi.git (wrap-ansi), https://github.com/sindresorhus/xdg-basedir.git (xdg-basedir). This software contains the following license and notice below:
MIT License
@@ -485,7 +432,7 @@
-----
-The following software may be included in this product: @svgr/babel-plugin-add-jsx-attribute, @svgr/babel-plugin-remove-jsx-attribute, @svgr/babel-plugin-remove-jsx-empty-expression, @svgr/babel-plugin-replace-jsx-attribute-value, @svgr/babel-plugin-svg-dynamic-title, @svgr/babel-plugin-svg-em-dimensions, @svgr/babel-plugin-transform-react-native-svg, @svgr/babel-plugin-transform-svg-component, @svgr/babel-preset, @svgr/core, @svgr/hast-util-to-babel-ast, @svgr/plugin-jsx, @svgr/plugin-svgo, @svgr/webpack. A copy of the source code may be downloaded from https://github.com/smooth-code/svgr/tree/master/packages/babel-plugin-add-jsx-attribute (@svgr/babel-plugin-add-jsx-attribute), https://github.com/smooth-code/svgr/tree/master/packages/babel-plugin-remove-jsx-attribute (@svgr/babel-plugin-remove-jsx-attribute), https://github.com/smooth-code/svgr/tree/master/packages/babel-plugin-remove-jsx-empty-expression (@svgr/babel-plugin-remove-jsx-empty-expression), https://github.com/smooth-code/svgr/tree/master/packages/babel-plugin-replace-jsx-attribute-value (@svgr/babel-plugin-replace-jsx-attribute-value), https://github.com/smooth-code/svgr/tree/master/packages/babel-plugin-svg-dynamic-title (@svgr/babel-plugin-svg-dynamic-title), https://github.com/smooth-code/svgr/tree/master/packages/babel-plugin-svg-em-dimensions (@svgr/babel-plugin-svg-em-dimensions), https://github.com/smooth-code/svgr/tree/master/packages/babel-plugin-transform-react-native-svg (@svgr/babel-plugin-transform-react-native-svg), https://github.com/smooth-code/svgr/tree/master/packages/babel-plugin-transform-svg-component (@svgr/babel-plugin-transform-svg-component), https://github.com/smooth-code/svgr/tree/master/packages/babel-preset (@svgr/babel-preset), https://github.com/smooth-code/svgr/tree/master/packages/core (@svgr/core), https://github.com/smooth-code/svgr/tree/master/packages/hast-util-to-babel-ast (@svgr/hast-util-to-babel-ast), https://github.com/smooth-code/svgr/tree/master/packages/plugin-jsx (@svgr/plugin-jsx), https://github.com/smooth-code/svgr/tree/master/packages/plugin-svgo (@svgr/plugin-svgo), git@github.com:smooth-code/svgr.git (@svgr/webpack). This software contains the following license and notice below:
+The following software may be included in this product: @svgr/babel-plugin-add-jsx-attribute, @svgr/babel-plugin-remove-jsx-attribute, @svgr/babel-plugin-remove-jsx-empty-expression, @svgr/babel-plugin-replace-jsx-attribute-value, @svgr/babel-plugin-svg-dynamic-title, @svgr/babel-plugin-svg-em-dimensions, @svgr/babel-plugin-transform-react-native-svg, @svgr/babel-plugin-transform-svg-component, @svgr/babel-preset, @svgr/core, @svgr/hast-util-to-babel-ast, @svgr/plugin-jsx, @svgr/plugin-svgo, @svgr/webpack. A copy of the source code may be downloaded from https://github.com/smooth-code/svgr/tree/master/packages/babel-plugin-add-jsx-attribute (@svgr/babel-plugin-add-jsx-attribute), https://github.com/smooth-code/svgr/tree/master/packages/babel-plugin-remove-jsx-attribute (@svgr/babel-plugin-remove-jsx-attribute), https://github.com/smooth-code/svgr/tree/master/packages/babel-plugin-remove-jsx-empty-expression (@svgr/babel-plugin-remove-jsx-empty-expression), https://github.com/smooth-code/svgr/tree/master/packages/babel-plugin-replace-jsx-attribute-value (@svgr/babel-plugin-replace-jsx-attribute-value), https://github.com/smooth-code/svgr/tree/master/packages/babel-plugin-svg-dynamic-title (@svgr/babel-plugin-svg-dynamic-title), https://github.com/smooth-code/svgr/tree/master/packages/babel-plugin-svg-em-dimensions (@svgr/babel-plugin-svg-em-dimensions), https://github.com/smooth-code/svgr/tree/master/packages/babel-plugin-transform-react-native-svg (@svgr/babel-plugin-transform-react-native-svg), https://github.com/smooth-code/svgr/tree/master/packages/babel-plugin-transform-svg-component (@svgr/babel-plugin-transform-svg-component), https://github.com/smooth-code/svgr/tree/master/packages/babel-preset (@svgr/babel-preset), https://github.com/smooth-code/svgr/tree/master/packages/core (@svgr/core), https://github.com/smooth-code/svgr/tree/master/packages/hast-util-to-babel-ast (@svgr/hast-util-to-babel-ast), https://github.com/smooth-code/svgr/tree/master/packages/plugin-jsx (@svgr/plugin-jsx), https://github.com/smooth-code/svgr/tree/master/packages/plugin-svgo (@svgr/plugin-svgo), https://github.com/smooth-code/svgr/tree/master/packages/webpack (@svgr/webpack). This software contains the following license and notice below:
Copyright 2017 Smooth Code
@@ -523,7 +470,7 @@
-----
-The following software may be included in this product: @types/babel__core, @types/babel__generator, @types/babel__template, @types/babel__traverse, @types/history, @types/istanbul-lib-coverage, @types/istanbul-lib-report, @types/istanbul-reports, @types/jest, @types/jest-diff, @types/node, @types/prop-types, @types/q, @types/react, @types/react-dom, @types/react-router, @types/react-router-dom, @types/react-slick, @types/stack-utils, @types/yargs, @types/yargs-parser. A copy of the source code may be downloaded from https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/babel__core), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/babel__generator), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/babel__template), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/babel__traverse), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/history), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/istanbul-lib-coverage), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/istanbul-lib-report), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/istanbul-reports), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/jest), https://github.com/facebook/jest/tree/master/packages/jest-diff (@types/jest-diff), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/node), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/prop-types), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/q), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/react), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/react-dom), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/react-router), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/react-router-dom), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/react-slick), https://www.github.com/DefinitelyTyped/DefinitelyTyped.git (@types/stack-utils), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/yargs), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/yargs-parser). This software contains the following license and notice below:
+The following software may be included in this product: @types/babel__core, @types/babel__generator, @types/babel__template, @types/babel__traverse, @types/classnames, @types/color-name, @types/eslint-visitor-keys, @types/events, @types/glob, @types/history, @types/istanbul-lib-coverage, @types/istanbul-lib-report, @types/istanbul-reports, @types/jest, @types/jest-diff, @types/json-schema, @types/minimatch, @types/node, @types/normalize-package-data, @types/parse-json, @types/prop-types, @types/q, @types/react, @types/react-dom, @types/react-router, @types/react-router-dom, @types/react-slick, @types/stack-utils, @types/yargs, @types/yargs-parser. A copy of the source code may be downloaded from https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/babel__core), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/babel__generator), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/babel__template), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/babel__traverse), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/classnames), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/color-name), https://www.github.com/DefinitelyTyped/DefinitelyTyped.git (@types/eslint-visitor-keys), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/events), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/glob), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/history), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/istanbul-lib-coverage), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/istanbul-lib-report), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/istanbul-reports), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/jest), https://github.com/facebook/jest/tree/master/packages/jest-diff (@types/jest-diff), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/json-schema), https://www.github.com/DefinitelyTyped/DefinitelyTyped.git (@types/minimatch), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/node), https://www.github.com/DefinitelyTyped/DefinitelyTyped.git (@types/normalize-package-data), https://www.github.com/DefinitelyTyped/DefinitelyTyped.git (@types/parse-json), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/prop-types), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/q), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/react), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/react-dom), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/react-router), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/react-router-dom), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/react-slick), https://www.github.com/DefinitelyTyped/DefinitelyTyped.git (@types/stack-utils), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/yargs), https://github.com/DefinitelyTyped/DefinitelyTyped.git (@types/yargs-parser). This software contains the following license and notice below:
MIT License
@@ -549,7 +496,33 @@
-----
-The following software may be included in this product: @typescript-eslint/eslint-plugin. A copy of the source code may be downloaded from https://github.com/typescript-eslint/typescript-eslint.git. This software contains the following license and notice below:
+The following software may be included in this product: @types/node. A copy of the source code may be downloaded from https://github.com/DefinitelyTyped/DefinitelyTyped.git. This software contains the following license and notice below:
+
+MIT License
+
+ Copyright (c) Microsoft Corporation.
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE
+
+-----
+
+The following software may be included in this product: @typescript-eslint/eslint-plugin, @typescript-eslint/experimental-utils. A copy of the source code may be downloaded from https://github.com/typescript-eslint/typescript-eslint.git (@typescript-eslint/eslint-plugin), https://github.com/typescript-eslint/typescript-eslint.git (@typescript-eslint/experimental-utils). This software contains the following license and notice below:
MIT License
@@ -1193,32 +1166,6 @@
-----
-The following software may be included in this product: acorn-dynamic-import. A copy of the source code may be downloaded from https://github.com/kesne/acorn-dynamic-import. This software contains the following license and notice below:
-
-MIT License
-
-Copyright (c) 2016 Jordan Gensler
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
------
-
The following software may be included in this product: acorn-globals, is-promise. A copy of the source code may be downloaded from https://github.com/ForbesLindesay/acorn-globals.git (acorn-globals), https://github.com/then/is-promise.git (is-promise). This software contains the following license and notice below:
Copyright (c) 2014 Forbes Lindesay
@@ -1457,6 +1404,20 @@
-----
+The following software may be included in this product: ansi-escapes, eslint-plugin-unicorn, figures, find-cache-dir, globals, xo. A copy of the source code may be downloaded from https://github.com/sindresorhus/ansi-escapes.git (ansi-escapes), https://github.com/sindresorhus/eslint-plugin-unicorn.git (eslint-plugin-unicorn), https://github.com/sindresorhus/figures.git (figures), https://github.com/avajs/find-cache-dir.git (find-cache-dir), https://github.com/sindresorhus/globals.git (globals), https://github.com/xojs/xo.git (xo). This software contains the following license and notice below:
+
+MIT License
+
+Copyright (c) Sindre Sorhus <sindresorhus@gmail.com> (https://sindresorhus.com)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+-----
+
The following software may be included in this product: ansi-html. A copy of the source code may be downloaded from git://github.com/Tjatse/ansi-html.git. This software contains the following license and notice below:
Apache License
@@ -1663,7 +1624,7 @@
-----
-The following software may be included in this product: ansi-regex, ansi-styles, array-union, array-uniq, arrify, caller-callsite, caller-path, callsites, camelcase, chalk, cli-cursor, code-point-at, crypto-random-string, decamelize, detect-newline, dot-prop, escape-string-regexp, figures, find-up, get-stream, globby, has-ansi, import-cwd, import-fresh, import-from, import-lazy, ip-regex, is-absolute-url, is-binary-path, is-fullwidth-code-point, is-obj, is-path-in-cwd, is-path-inside, is-stream, is-wsl, load-json-file, locate-path, lowercase-keys, npm-run-path, number-is-nan, object-assign, onetime, os-tmpdir, p-defer, p-each-series, p-finally, p-locate, p-reduce, p-try, parse-json, path-exists, path-is-absolute, path-key, path-type, pify, pkg-dir, pkg-up, read-pkg, read-pkg-up, resolve-cwd, resolve-from, restore-cursor, semver-diff, shebang-regex, string-width, strip-ansi, strip-bom, strip-eof, strip-json-comments, supports-color, term-size, unique-string, wrap-ansi, xdg-basedir. A copy of the source code may be downloaded from https://github.com/chalk/ansi-regex.git (ansi-regex), https://github.com/chalk/ansi-styles.git (ansi-styles), https://github.com/sindresorhus/array-union.git (array-union), https://github.com/sindresorhus/array-uniq.git (array-uniq), https://github.com/sindresorhus/arrify.git (arrify), https://github.com/sindresorhus/caller-callsite.git (caller-callsite), https://github.com/sindresorhus/caller-path.git (caller-path), https://github.com/sindresorhus/callsites.git (callsites), https://github.com/sindresorhus/camelcase.git (camelcase), https://github.com/chalk/chalk.git (chalk), https://github.com/sindresorhus/cli-cursor.git (cli-cursor), https://github.com/sindresorhus/code-point-at.git (code-point-at), https://github.com/sindresorhus/crypto-random-string.git (crypto-random-string), https://github.com/sindresorhus/decamelize.git (decamelize), https://github.com/sindresorhus/detect-newline.git (detect-newline), https://github.com/sindresorhus/dot-prop.git (dot-prop), https://github.com/sindresorhus/escape-string-regexp.git (escape-string-regexp), https://github.com/sindresorhus/figures.git (figures), https://github.com/sindresorhus/find-up.git (find-up), https://github.com/sindresorhus/get-stream.git (get-stream), https://github.com/sindresorhus/globby.git (globby), https://github.com/sindresorhus/has-ansi.git (has-ansi), https://github.com/sindresorhus/import-cwd.git (import-cwd), https://github.com/sindresorhus/import-fresh.git (import-fresh), https://github.com/sindresorhus/import-from.git (import-from), https://github.com/sindresorhus/import-lazy.git (import-lazy), https://github.com/sindresorhus/ip-regex.git (ip-regex), https://github.com/sindresorhus/is-absolute-url.git (is-absolute-url), https://github.com/sindresorhus/is-binary-path.git (is-binary-path), https://github.com/sindresorhus/is-fullwidth-code-point.git (is-fullwidth-code-point), https://github.com/sindresorhus/is-obj.git (is-obj), https://github.com/sindresorhus/is-path-in-cwd.git (is-path-in-cwd), https://github.com/sindresorhus/is-path-inside.git (is-path-inside), https://github.com/sindresorhus/is-stream.git (is-stream), https://github.com/sindresorhus/is-wsl.git (is-wsl), https://github.com/sindresorhus/load-json-file.git (load-json-file), https://github.com/sindresorhus/locate-path.git (locate-path), https://github.com/sindresorhus/lowercase-keys.git (lowercase-keys), https://github.com/sindresorhus/npm-run-path.git (npm-run-path), https://github.com/sindresorhus/number-is-nan.git (number-is-nan), https://github.com/sindresorhus/object-assign.git (object-assign), https://github.com/sindresorhus/onetime.git (onetime), https://github.com/sindresorhus/os-tmpdir.git (os-tmpdir), https://github.com/sindresorhus/p-defer.git (p-defer), https://github.com/sindresorhus/p-each-series.git (p-each-series), https://github.com/sindresorhus/p-finally.git (p-finally), https://github.com/sindresorhus/p-locate.git (p-locate), https://github.com/sindresorhus/p-reduce.git (p-reduce), https://github.com/sindresorhus/p-try.git (p-try), https://github.com/sindresorhus/parse-json.git (parse-json), https://github.com/sindresorhus/path-exists.git (path-exists), https://github.com/sindresorhus/path-is-absolute.git (path-is-absolute), https://github.com/sindresorhus/path-key.git (path-key), https://github.com/sindresorhus/path-type.git (path-type), https://github.com/sindresorhus/pify.git (pify), https://github.com/sindresorhus/pkg-dir.git (pkg-dir), https://github.com/sindresorhus/pkg-up.git (pkg-up), https://github.com/sindresorhus/read-pkg.git (read-pkg), https://github.com/sindresorhus/read-pkg-up.git (read-pkg-up), https://github.com/sindresorhus/resolve-cwd.git (resolve-cwd), https://github.com/sindresorhus/resolve-from.git (resolve-from), https://github.com/sindresorhus/restore-cursor.git (restore-cursor), https://github.com/sindresorhus/semver-diff.git (semver-diff), https://github.com/sindresorhus/shebang-regex.git (shebang-regex), https://github.com/sindresorhus/string-width.git (string-width), https://github.com/chalk/strip-ansi.git (strip-ansi), https://github.com/sindresorhus/strip-bom.git (strip-bom), https://github.com/sindresorhus/strip-eof.git (strip-eof), https://github.com/sindresorhus/strip-json-comments.git (strip-json-comments), https://github.com/chalk/supports-color.git (supports-color), https://github.com/sindresorhus/term-size.git (term-size), https://github.com/sindresorhus/unique-string.git (unique-string), https://github.com/chalk/wrap-ansi.git (wrap-ansi), https://github.com/sindresorhus/xdg-basedir.git (xdg-basedir). This software contains the following license and notice below:
+The following software may be included in this product: ansi-regex, ansi-styles, array-find-index, array-union, array-uniq, arrify, buf-compare, caller-callsite, caller-path, callsites, camelcase, camelcase-keys, chalk, code-point-at, core-assert, crypto-random-string, decamelize, deep-strict-equal, detect-newline, dot-prop, escape-string-regexp, find-up, get-stream, globby, has-ansi, import-cwd, import-fresh, import-from, import-lazy, ip-regex, is-absolute-url, is-binary-path, is-fullwidth-code-point, is-obj, is-path-inside, is-plain-obj, is-stream, is-wsl, js-types, load-json-file, locate-path, loud-rejection, lowercase-keys, map-obj, normalize-url, npm-run-path, number-is-nan, object-assign, os-tmpdir, p-defer, p-each-series, p-finally, p-locate, p-reduce, p-try, parse-json, path-exists, path-is-absolute, path-key, path-type, pify, pkg-dir, pkg-up, prepend-http, query-string, read-pkg, read-pkg-up, redent, resolve-cwd, resolve-from, semver-diff, shebang-regex, sort-keys, string-width, strip-ansi, strip-bom, strip-eof, strip-indent, strip-json-comments, supports-color, term-size, trim-newlines, unique-string, wrap-ansi, xdg-basedir. A copy of the source code may be downloaded from https://github.com/chalk/ansi-regex.git (ansi-regex), https://github.com/chalk/ansi-styles.git (ansi-styles), https://github.com/sindresorhus/array-find-index.git (array-find-index), https://github.com/sindresorhus/array-union.git (array-union), https://github.com/sindresorhus/array-uniq.git (array-uniq), https://github.com/sindresorhus/arrify.git (arrify), https://github.com/sindresorhus/buf-compare.git (buf-compare), https://github.com/sindresorhus/caller-callsite.git (caller-callsite), https://github.com/sindresorhus/caller-path.git (caller-path), https://github.com/sindresorhus/callsites.git (callsites), https://github.com/sindresorhus/camelcase.git (camelcase), https://github.com/sindresorhus/camelcase-keys.git (camelcase-keys), https://github.com/chalk/chalk.git (chalk), https://github.com/sindresorhus/code-point-at.git (code-point-at), https://github.com/sindresorhus/core-assert.git (core-assert), https://github.com/sindresorhus/crypto-random-string.git (crypto-random-string), https://github.com/sindresorhus/decamelize.git (decamelize), https://github.com/sindresorhus/deep-strict-equal.git (deep-strict-equal), https://github.com/sindresorhus/detect-newline.git (detect-newline), https://github.com/sindresorhus/dot-prop.git (dot-prop), https://github.com/sindresorhus/escape-string-regexp.git (escape-string-regexp), https://github.com/sindresorhus/find-up.git (find-up), https://github.com/sindresorhus/get-stream.git (get-stream), https://github.com/sindresorhus/globby.git (globby), https://github.com/sindresorhus/has-ansi.git (has-ansi), https://github.com/sindresorhus/import-cwd.git (import-cwd), https://github.com/sindresorhus/import-fresh.git (import-fresh), https://github.com/sindresorhus/import-from.git (import-from), https://github.com/sindresorhus/import-lazy.git (import-lazy), https://github.com/sindresorhus/ip-regex.git (ip-regex), https://github.com/sindresorhus/is-absolute-url.git (is-absolute-url), https://github.com/sindresorhus/is-binary-path.git (is-binary-path), https://github.com/sindresorhus/is-fullwidth-code-point.git (is-fullwidth-code-point), https://github.com/sindresorhus/is-obj.git (is-obj), https://github.com/sindresorhus/is-path-inside.git (is-path-inside), https://github.com/sindresorhus/is-plain-obj.git (is-plain-obj), https://github.com/sindresorhus/is-stream.git (is-stream), https://github.com/sindresorhus/is-wsl.git (is-wsl), https://github.com/sindresorhus/js-types.git (js-types), https://github.com/sindresorhus/load-json-file.git (load-json-file), https://github.com/sindresorhus/locate-path.git (locate-path), https://github.com/sindresorhus/loud-rejection.git (loud-rejection), https://github.com/sindresorhus/lowercase-keys.git (lowercase-keys), https://github.com/sindresorhus/map-obj.git (map-obj), https://github.com/sindresorhus/normalize-url.git (normalize-url), https://github.com/sindresorhus/npm-run-path.git (npm-run-path), https://github.com/sindresorhus/number-is-nan.git (number-is-nan), https://github.com/sindresorhus/object-assign.git (object-assign), https://github.com/sindresorhus/os-tmpdir.git (os-tmpdir), https://github.com/sindresorhus/p-defer.git (p-defer), https://github.com/sindresorhus/p-each-series.git (p-each-series), https://github.com/sindresorhus/p-finally.git (p-finally), https://github.com/sindresorhus/p-locate.git (p-locate), https://github.com/sindresorhus/p-reduce.git (p-reduce), https://github.com/sindresorhus/p-try.git (p-try), https://github.com/sindresorhus/parse-json.git (parse-json), https://github.com/sindresorhus/path-exists.git (path-exists), https://github.com/sindresorhus/path-is-absolute.git (path-is-absolute), https://github.com/sindresorhus/path-key.git (path-key), https://github.com/sindresorhus/path-type.git (path-type), https://github.com/sindresorhus/pify.git (pify), https://github.com/sindresorhus/pkg-dir.git (pkg-dir), https://github.com/sindresorhus/pkg-up.git (pkg-up), https://github.com/sindresorhus/prepend-http.git (prepend-http), https://github.com/sindresorhus/query-string.git (query-string), https://github.com/sindresorhus/read-pkg.git (read-pkg), https://github.com/sindresorhus/read-pkg-up.git (read-pkg-up), https://github.com/sindresorhus/redent.git (redent), https://github.com/sindresorhus/resolve-cwd.git (resolve-cwd), https://github.com/sindresorhus/resolve-from.git (resolve-from), https://github.com/sindresorhus/semver-diff.git (semver-diff), https://github.com/sindresorhus/shebang-regex.git (shebang-regex), https://github.com/sindresorhus/sort-keys.git (sort-keys), https://github.com/sindresorhus/string-width.git (string-width), https://github.com/chalk/strip-ansi.git (strip-ansi), https://github.com/sindresorhus/strip-bom.git (strip-bom), https://github.com/sindresorhus/strip-eof.git (strip-eof), https://github.com/sindresorhus/strip-indent.git (strip-indent), https://github.com/sindresorhus/strip-json-comments.git (strip-json-comments), https://github.com/chalk/supports-color.git (supports-color), https://github.com/sindresorhus/term-size.git (term-size), https://github.com/sindresorhus/trim-newlines.git (trim-newlines), https://github.com/sindresorhus/unique-string.git (unique-string), https://github.com/chalk/wrap-ansi.git (wrap-ansi), https://github.com/sindresorhus/xdg-basedir.git (xdg-basedir). This software contains the following license and notice below:
The MIT License (MIT)
@@ -1736,6 +1697,26 @@
-----
+The following software may be included in this product: anymatch. A copy of the source code may be downloaded from https://github.com/micromatch/anymatch. This software contains the following license and notice below:
+
+The ISC License
+
+Copyright (c) 2019 Elan Shanker, Paul Miller (https://paulmillr.com)
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
+IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+-----
+
The following software may be included in this product: aproba. A copy of the source code may be downloaded from https://github.com/iarna/aproba. This software contains the following license and notice below:
Copyright (c) 2015, Rebecca Turner <me@re-becca.org>
@@ -1986,6 +1967,32 @@
-----
+The following software may be included in this product: arity-n. A copy of the source code may be downloaded from http://github.com/stoeffel/arityN. This software contains the following license and notice below:
+
+The MIT License (MIT)
+
+Copyright (c) 2015 Christoph Hermann
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+-----
+
The following software may be included in this product: arr-diff, fill-range, for-in, has-value, has-values, kind-of, normalize-path, set-value, word-wrap. A copy of the source code may be downloaded from https://github.com/jonschlinkert/arr-diff.git (arr-diff), https://github.com/jonschlinkert/fill-range.git (fill-range), https://github.com/jonschlinkert/for-in.git (for-in), https://github.com/jonschlinkert/has-value.git (has-value), https://github.com/jonschlinkert/has-values.git (has-values), https://github.com/jonschlinkert/kind-of.git (kind-of), https://github.com/jonschlinkert/normalize-path.git (normalize-path), https://github.com/jonschlinkert/set-value.git (set-value), https://github.com/jonschlinkert/word-wrap.git (word-wrap). This software contains the following license and notice below:
The MIT License (MIT)
@@ -2012,7 +2019,7 @@
-----
-The following software may be included in this product: arr-flatten, clone-deep, is-glob, is-plain-object, kind-of. A copy of the source code may be downloaded from https://github.com/jonschlinkert/arr-flatten.git (arr-flatten), https://github.com/jonschlinkert/clone-deep.git (clone-deep), https://github.com/micromatch/is-glob.git (is-glob), https://github.com/jonschlinkert/is-plain-object.git (is-plain-object), https://github.com/jonschlinkert/kind-of.git (kind-of). This software contains the following license and notice below:
+The following software may be included in this product: arr-flatten, is-glob, is-plain-object, is-relative, kind-of. A copy of the source code may be downloaded from https://github.com/jonschlinkert/arr-flatten.git (arr-flatten), https://github.com/micromatch/is-glob.git (is-glob), https://github.com/jonschlinkert/is-plain-object.git (is-plain-object), https://github.com/jonschlinkert/is-relative.git (is-relative), https://github.com/jonschlinkert/kind-of.git (kind-of). This software contains the following license and notice below:
The MIT License (MIT)
@@ -2090,7 +2097,33 @@
-----
-The following software may be included in this product: array-flatten, camel-case, lower-case, no-case, param-case, path-to-regexp, upper-case. A copy of the source code may be downloaded from git://github.com/blakeembrey/array-flatten.git (array-flatten), git://github.com/blakeembrey/camel-case.git (camel-case), git://github.com/blakeembrey/lower-case.git (lower-case), git://github.com/blakeembrey/no-case.git (no-case), git://github.com/blakeembrey/param-case.git (param-case), https://github.com/component/path-to-regexp.git (path-to-regexp), git://github.com/blakeembrey/upper-case.git (upper-case). This software contains the following license and notice below:
+The following software may be included in this product: array-find. A copy of the source code may be downloaded from https://github.com/stefanduberg/array-find.git. This software contains the following license and notice below:
+
+The MIT License
+
+Copyright (c) Stefan Duberg
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+-----
+
+The following software may be included in this product: array-flatten, camel-case, dot-case, lower-case, no-case, param-case, pascal-case, path-to-regexp. A copy of the source code may be downloaded from git://github.com/blakeembrey/array-flatten.git (array-flatten), git://github.com/blakeembrey/change-case.git (camel-case), git://github.com/blakeembrey/change-case.git (dot-case), git://github.com/blakeembrey/change-case.git (lower-case), git://github.com/blakeembrey/change-case.git (no-case), git://github.com/blakeembrey/change-case.git (param-case), git://github.com/blakeembrey/change-case.git (pascal-case), https://github.com/component/path-to-regexp.git (path-to-regexp). This software contains the following license and notice below:
The MIT License (MIT)
@@ -2168,6 +2201,32 @@
-----
+The following software may be included in this product: array.prototype.flat. A copy of the source code may be downloaded from git://github.com/es-shims/Array.prototype.flat.git. This software contains the following license and notice below:
+
+MIT License
+
+Copyright (c) 2017 ECMAScript Shims
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+-----
+
The following software may be included in this product: asap. A copy of the source code may be downloaded from https://github.com/kriskowal/asap.git. This software contains the following license and notice below:
Copyright 2009–2014 Contributors. All rights reserved.
@@ -2239,7 +2298,7 @@
-----
-The following software may be included in this product: assign-symbols, contains-path, define-property, is-accessor-descriptor, is-data-descriptor, is-extendable, lazy-cache, pascalcase, shallow-clone. A copy of the source code may be downloaded from https://github.com/jonschlinkert/assign-symbols.git (assign-symbols), https://github.com/jonschlinkert/contains-path.git (contains-path), https://github.com/jonschlinkert/define-property.git (define-property), https://github.com/jonschlinkert/is-accessor-descriptor.git (is-accessor-descriptor), https://github.com/jonschlinkert/is-data-descriptor.git (is-data-descriptor), https://github.com/jonschlinkert/is-extendable.git (is-extendable), https://github.com/jonschlinkert/lazy-cache.git (lazy-cache), https://github.com/jonschlinkert/pascalcase.git (pascalcase), https://github.com/jonschlinkert/shallow-clone.git (shallow-clone). This software contains the following license and notice below:
+The following software may be included in this product: assign-symbols, contains-path, define-property, is-accessor-descriptor, is-data-descriptor, is-extendable, lazy-cache, pascalcase, shallow-clone, unc-path-regex. A copy of the source code may be downloaded from https://github.com/jonschlinkert/assign-symbols.git (assign-symbols), https://github.com/jonschlinkert/contains-path.git (contains-path), https://github.com/jonschlinkert/define-property.git (define-property), https://github.com/jonschlinkert/is-accessor-descriptor.git (is-accessor-descriptor), https://github.com/jonschlinkert/is-data-descriptor.git (is-data-descriptor), https://github.com/jonschlinkert/is-extendable.git (is-extendable), https://github.com/jonschlinkert/lazy-cache.git (lazy-cache), https://github.com/jonschlinkert/pascalcase.git (pascalcase), https://github.com/jonschlinkert/shallow-clone.git (shallow-clone), https://github.com/regexhq/unc-path-regex.git (unc-path-regex). This software contains the following license and notice below:
The MIT License (MIT)
@@ -2265,7 +2324,7 @@
-----
-The following software may be included in this product: astral-regex, dir-glob. A copy of the source code may be downloaded from https://github.com/kevva/astral-regex.git (astral-regex), https://github.com/kevva/dir-glob.git (dir-glob). This software contains the following license and notice below:
+The following software may be included in this product: astral-regex, dir-glob, shebang-command. A copy of the source code may be downloaded from https://github.com/kevva/astral-regex.git (astral-regex), https://github.com/kevva/dir-glob.git (dir-glob), https://github.com/kevva/shebang-command.git (shebang-command). This software contains the following license and notice below:
MIT License
@@ -2342,6 +2401,17 @@
-----
+The following software may be included in this product: at-least-node. A copy of the source code may be downloaded from git+https://github.com/RyanZim/at-least-node.git. This software contains the following license and notice below:
+
+The ISC License
+Copyright (c) 2020 Ryan Zimmerman <opensrc@ryanzim.com>
+
+Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+-----
+
The following software may be included in this product: atob. A copy of the source code may be downloaded from git://git.coolaj86.com/coolaj86/atob.js.git. This software contains the following license and notice below:
At your option you may choose either of the following licenses:
@@ -2703,37 +2773,9 @@
-----
-The following software may be included in this product: babel-jest, jest, jest-resolve. A copy of the source code may be downloaded from https://github.com/facebook/jest.git (babel-jest), https://github.com/facebook/jest (jest), https://github.com/facebook/jest.git (jest-resolve). This software contains the following license and notice below:
-
-MIT License
-
-For Jest software
-
-Copyright (c) 2014-present, Facebook, Inc.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
------
-
The following software may be included in this product: babel-loader. A copy of the source code may be downloaded from https://github.com/babel/babel-loader.git. This software contains the following license and notice below:
-Copyright (c) 2014-2016 Luís Couto <hello@luiscouto.pt>
+Copyright (c) 2014-2019 Luís Couto <hello@luiscouto.pt>
MIT License
@@ -3098,6 +3140,20 @@
-----
+The following software may be included in this product: binary-extensions, is-binary-path. A copy of the source code may be downloaded from https://github.com/sindresorhus/binary-extensions.git (binary-extensions), https://github.com/sindresorhus/is-binary-path.git (is-binary-path). This software contains the following license and notice below:
+
+MIT License
+
+Copyright (c) 2019 Sindre Sorhus <sindresorhus@gmail.com> (https://sindresorhus.com), Paul Miller (https://paulmillr.com)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+-----
+
The following software may be included in this product: bindings. A copy of the source code may be downloaded from git://github.com/TooTallNate/node-bindings.git. This software contains the following license and notice below:
(The MIT License)
@@ -3231,7 +3287,7 @@
-----
-The following software may be included in this product: braces, micromatch, normalize-path. A copy of the source code may be downloaded from https://github.com/micromatch/braces.git (braces), https://github.com/micromatch/micromatch.git (micromatch), https://github.com/jonschlinkert/normalize-path.git (normalize-path). This software contains the following license and notice below:
+The following software may be included in this product: braces, clone-deep, micromatch, normalize-path. A copy of the source code may be downloaded from https://github.com/micromatch/braces.git (braces), https://github.com/jonschlinkert/clone-deep.git (clone-deep), https://github.com/micromatch/micromatch.git (micromatch), https://github.com/jonschlinkert/normalize-path.git (normalize-path). This software contains the following license and notice below:
The MIT License (MIT)
@@ -3498,6 +3554,31 @@
The MIT License (MIT)
+Copyright 2014 Andrey Sitnik <andrey@sitnik.ru> and other contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+-----
+
+The following software may be included in this product: browserslist. A copy of the source code may be downloaded from https://github.com/browserslist/browserslist.git. This software contains the following license and notice below:
+
+The MIT License (MIT)
+
Copyright 2014 Andrey Sitnik <andrey@sitnik.ru>
Permission is hereby granted, free of charge, to any person obtaining a copy of
@@ -4284,7 +4365,33 @@
-----
-The following software may be included in this product: chownr, fs-write-stream-atomic, ini, isexe, json-stringify-safe, lru-cache, minimatch, mute-stream, once, pseudomap, rimraf, semver, server-destroy, which, wrappy, yallist. A copy of the source code may be downloaded from git://github.com/isaacs/chownr.git (chownr), https://github.com/npm/fs-write-stream-atomic (fs-write-stream-atomic), git://github.com/isaacs/ini.git (ini), git+https://github.com/isaacs/isexe.git (isexe), git://github.com/isaacs/json-stringify-safe (json-stringify-safe), git://github.com/isaacs/node-lru-cache.git (lru-cache), git://github.com/isaacs/minimatch.git (minimatch), git://github.com/isaacs/mute-stream (mute-stream), git://github.com/isaacs/once (once), git+https://github.com/isaacs/pseudomap.git (pseudomap), git://github.com/isaacs/rimraf.git (rimraf), https://github.com/npm/node-semver (semver), git://github.com/isaacs/server-destroy (server-destroy), git://github.com/isaacs/node-which.git (which), https://github.com/npm/wrappy (wrappy), git+https://github.com/isaacs/yallist.git (yallist). This software contains the following license and notice below:
+The following software may be included in this product: chokidar. A copy of the source code may be downloaded from git+https://github.com/paulmillr/chokidar.git. This software contains the following license and notice below:
+
+The MIT License (MIT)
+
+Copyright (c) 2012-2019 Paul Miller (https://paulmillr.com), Elan Shanker
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the “Software”), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+-----
+
+The following software may be included in this product: chownr, fs-minipass, fs-write-stream-atomic, ini, isexe, json-stringify-safe, lru-cache, minimatch, minipass-collect, minipass-flush, minipass-pipeline, mute-stream, once, pseudomap, rimraf, semver, server-destroy, which, wrappy, yallist. A copy of the source code may be downloaded from git://github.com/isaacs/chownr.git (chownr), git+https://github.com/npm/fs-minipass.git (fs-minipass), https://github.com/npm/fs-write-stream-atomic (fs-write-stream-atomic), git://github.com/isaacs/ini.git (ini), git+https://github.com/isaacs/isexe.git (isexe), git://github.com/isaacs/json-stringify-safe (json-stringify-safe), git://github.com/isaacs/node-lru-cache.git (lru-cache), git://github.com/isaacs/minimatch.git (minimatch), git+https://github.com/isaacs/minipass-flush.git (minipass-flush), git://github.com/isaacs/mute-stream (mute-stream), git://github.com/isaacs/once (once), git+https://github.com/isaacs/pseudomap.git (pseudomap), git://github.com/isaacs/rimraf.git (rimraf), https://github.com/npm/node-semver (semver), git://github.com/isaacs/server-destroy (server-destroy), git://github.com/isaacs/node-which.git (which), https://github.com/npm/wrappy (wrappy), git+https://github.com/isaacs/yallist.git (yallist). This software contains the following license and notice below:
The ISC License
@@ -4459,6 +4566,20 @@
-----
+The following software may be included in this product: clean-regexp, map-age-cleaner. A copy of the source code may be downloaded from https://github.com/SamVerschueren/clean-regexp.git (clean-regexp), https://github.com/SamVerschueren/map-age-cleaner.git (map-age-cleaner). This software contains the following license and notice below:
+
+MIT License
+
+Copyright (c) Sam Verschueren <sam.verschueren@gmail.com> (github.com/SamVerschueren)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+-----
+
The following software may be included in this product: cli-width. A copy of the source code may be downloaded from git@github.com:knownasilya/cli-width.git. This software contains the following license and notice below:
Copyright (c) 2015, Ilya Radchenko <ilya@burstcreations.com>
@@ -4836,6 +4957,18 @@
-----
+The following software may be included in this product: compose-function. A copy of the source code may be downloaded from http://github.com/stoeffel/compose-function. This software contains the following license and notice below:
+
+Copyright © 2015 Christoph Hermann
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+-----
+
The following software may be included in this product: compressible. A copy of the source code may be downloaded from https://github.com/jshttp/compressible.git. This software contains the following license and notice below:
(The MIT License)
@@ -5101,7 +5234,7 @@
-----
-The following software may be included in this product: copy-descriptor, expand-brackets. A copy of the source code may be downloaded from https://github.com/jonschlinkert/copy-descriptor.git (copy-descriptor), https://github.com/jonschlinkert/expand-brackets.git (expand-brackets). This software contains the following license and notice below:
+The following software may be included in this product: copy-descriptor, expand-brackets, to-absolute-glob. A copy of the source code may be downloaded from https://github.com/jonschlinkert/copy-descriptor.git (copy-descriptor), https://github.com/jonschlinkert/expand-brackets.git (expand-brackets), https://github.com/jonschlinkert/to-absolute-glob.git (to-absolute-glob). This software contains the following license and notice below:
The MIT License (MIT)
@@ -5407,6 +5540,20 @@
-----
+The following software may be included in this product: css. A copy of the source code may be downloaded from https://github.com/reworkcss/css.git. This software contains the following license and notice below:
+
+(The MIT License)
+
+Copyright (c) 2012 TJ Holowaychuk <tj@vision-media.ca>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the 'Software'), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+-----
+
The following software may be included in this product: css-declaration-sorter. A copy of the source code may be downloaded from https://github.com/Siilwyn/css-declaration-sorter.git. This software contains the following license and notice below:
The MIT License (MIT)
@@ -5432,7 +5579,7 @@
-----
-The following software may be included in this product: css-loader, enhanced-resolve, file-loader, html-webpack-plugin, less-loader, loader-utils, memory-fs, mini-css-extract-plugin, schema-utils, style-loader, terser-webpack-plugin, url-loader, watchpack, webpack, webpack-dev-middleware, webpack-dev-server. A copy of the source code may be downloaded from https://github.com/webpack-contrib/css-loader.git (css-loader), git://github.com/webpack/enhanced-resolve.git (enhanced-resolve), https://github.com/webpack-contrib/file-loader.git (file-loader), https://github.com/jantimon/html-webpack-plugin.git (html-webpack-plugin), https://github.com/webpack-contrib/less-loader.git (less-loader), https://github.com/webpack/loader-utils.git (loader-utils), https://github.com/webpack/memory-fs.git (memory-fs), https://github.com/webpack-contrib/mini-css-extract-plugin.git (mini-css-extract-plugin), https://github.com/webpack-contrib/schema-utils (schema-utils), https://github.com/webpack-contrib/style-loader.git (style-loader), https://github.com/webpack-contrib/terser-webpack-plugin.git (terser-webpack-plugin), https://github.com/webpack-contrib/url-loader.git (url-loader), https://github.com/webpack/watchpack.git (watchpack), https://github.com/webpack/webpack.git (webpack), https://github.com/webpack/webpack-dev-middleware.git (webpack-dev-middleware), https://github.com/webpack/webpack-dev-server.git (webpack-dev-server). This software contains the following license and notice below:
+The following software may be included in this product: css-loader, enhanced-resolve, eslint-loader, file-loader, html-webpack-plugin, less-loader, loader-utils, memory-fs, mini-css-extract-plugin, schema-utils, style-loader, terser-webpack-plugin, url-loader, watchpack, webpack, webpack-dev-middleware, webpack-dev-server. A copy of the source code may be downloaded from https://github.com/webpack-contrib/css-loader.git (css-loader), git://github.com/webpack/enhanced-resolve.git (enhanced-resolve), https://github.com/webpack-contrib/eslint-loader.git (eslint-loader), https://github.com/webpack-contrib/file-loader.git (file-loader), https://github.com/jantimon/html-webpack-plugin.git (html-webpack-plugin), https://github.com/webpack-contrib/less-loader.git (less-loader), https://github.com/webpack/loader-utils.git (loader-utils), https://github.com/webpack/memory-fs.git (memory-fs), https://github.com/webpack-contrib/mini-css-extract-plugin.git (mini-css-extract-plugin), https://github.com/webpack-contrib/schema-utils (schema-utils), https://github.com/webpack-contrib/style-loader.git (style-loader), https://github.com/webpack-contrib/terser-webpack-plugin.git (terser-webpack-plugin), https://github.com/webpack-contrib/url-loader.git (url-loader), https://github.com/webpack/watchpack.git (watchpack), https://github.com/webpack/webpack.git (webpack), https://github.com/webpack/webpack-dev-middleware.git (webpack-dev-middleware), https://github.com/webpack/webpack-dev-server.git (webpack-dev-server). This software contains the following license and notice below:
Copyright JS Foundation and other contributors
@@ -5733,6 +5880,32 @@
-----
+The following software may be included in this product: currently-unhandled, find-cache-dir, node-modules-regexp, normalize-range. A copy of the source code may be downloaded from https://github.com/jamestalmage/currently-unhandled.git (currently-unhandled), https://github.com/jamestalmage/find-cache-dir.git (find-cache-dir), https://github.com/jamestalmage/node-modules-regexp.git (node-modules-regexp), https://github.com/jamestalmage/normalize-range.git (normalize-range). This software contains the following license and notice below:
+
+The MIT License (MIT)
+
+Copyright (c) James Talmage <james@talmage.io> (github.com/jamestalmage)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+-----
+
The following software may be included in this product: customize-cra. A copy of the source code may be downloaded from https://github.com/arackaf/customize-cra.git. This software contains the following license and notice below:
Copyright 2018 and later Adam Rackis and contributors
@@ -5771,6 +5944,26 @@
-----
+The following software may be included in this product: d, es6-symbol. A copy of the source code may be downloaded from git://github.com/medikoo/d.git (d), git://github.com/medikoo/es6-symbol.git (es6-symbol). This software contains the following license and notice below:
+
+ISC License
+
+Copyright (c) 2013-2019, Mariusz Nowak, @medikoo, medikoo.com
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.
+
+-----
+
The following software may be included in this product: damerau-levenshtein. A copy of the source code may be downloaded from https://github.com/tad-lispy/node-damerau-levenshtein.git. This software contains the following license and notice below:
BSD 2-Clause License
@@ -5864,6 +6057,32 @@
-----
+The following software may be included in this product: decamelize-keys. A copy of the source code may be downloaded from https://github.com/dsblv/decamelize-keys.git. This software contains the following license and notice below:
+
+The MIT License (MIT)
+
+Copyright (c) Sindre Sorhus <sindresorhus@gmail.com> (sindresorhus.com), Dmirty Sobolev <disobolev@icloud.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+-----
+
The following software may be included in this product: decode-uri-component. A copy of the source code may be downloaded from https://github.com/SamVerschueren/decode-uri-component.git. This software contains the following license and notice below:
The MIT License (MIT)
@@ -6774,6 +6993,32 @@
-----
+The following software may be included in this product: enhance-visitors, eslint-ast-utils. A copy of the source code may be downloaded from https://github.com/jfmengels/enhance-visitors.git (enhance-visitors), https://github.com/jfmengels/eslint-ast-utils.git (eslint-ast-utils). This software contains the following license and notice below:
+
+The MIT License (MIT)
+
+Copyright (c) Jeroen Engels <jfm.engels@gmail.com> (github.com/jfmengels)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+-----
+
The following software may be included in this product: enquire.js. A copy of the source code may be downloaded from git://github.com/WickyNilliams/enquire.js.git. This software contains the following license and notice below:
The MIT License (MIT)
@@ -6826,7 +7071,7 @@
-----
-The following software may be included in this product: es-to-primitive, is-callable, is-date-object, is-string, is-symbol, object.getownpropertydescriptors, object.values, string.prototype.trimleft, string.prototype.trimright. A copy of the source code may be downloaded from git://github.com/ljharb/es-to-primitive.git (es-to-primitive), git://github.com/ljharb/is-callable.git (is-callable), git://github.com/ljharb/is-date-object.git (is-date-object), git://github.com/ljharb/is-string.git (is-string), git://github.com/inspect-js/is-symbol.git (is-symbol), git://github.com/es-shims/object.getownpropertydescriptors.git (object.getownpropertydescriptors), git://github.com/es-shims/Object.values.git (object.values), git://github.com/es-shims/String.prototype.trimLeft.git (string.prototype.trimleft), git://github.com/es-shims/String.prototype.trimRight.git (string.prototype.trimright). This software contains the following license and notice below:
+The following software may be included in this product: es-to-primitive, is-callable, is-date-object, is-string, is-symbol, object.entries, object.getownpropertydescriptors, object.values, string.prototype.matchall, string.prototype.trimleft, string.prototype.trimright. A copy of the source code may be downloaded from git://github.com/ljharb/es-to-primitive.git (es-to-primitive), git://github.com/ljharb/is-callable.git (is-callable), git://github.com/ljharb/is-date-object.git (is-date-object), git://github.com/ljharb/is-string.git (is-string), git://github.com/inspect-js/is-symbol.git (is-symbol), git://github.com/es-shims/Object.entries.git (object.entries), git://github.com/es-shims/object.getownpropertydescriptors.git (object.getownpropertydescriptors), git://github.com/es-shims/Object.values.git (object.values), git+https://github.com/ljharb/String.prototype.matchAll.git (string.prototype.matchall), git://github.com/es-shims/String.prototype.trimLeft.git (string.prototype.trimleft), git://github.com/es-shims/String.prototype.trimRight.git (string.prototype.trimright). This software contains the following license and notice below:
The MIT License (MIT)
@@ -6852,6 +7097,52 @@
-----
+The following software may be included in this product: es5-ext, ext. A copy of the source code may be downloaded from https://github.com/medikoo/es5-ext.git (es5-ext), https://github.com/medikoo/es5-ext/tree/ext (ext). This software contains the following license and notice below:
+
+ISC License
+
+Copyright (c) 2011-2019, Mariusz Nowak, @medikoo, medikoo.com
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.
+
+-----
+
+The following software may be included in this product: es6-iterator. A copy of the source code may be downloaded from git://github.com/medikoo/es6-iterator.git. This software contains the following license and notice below:
+
+The MIT License (MIT)
+
+Copyright (C) 2013-2017 Mariusz Nowak (www.medikoo.com)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+-----
+
The following software may be included in this product: escape-html. A copy of the source code may be downloaded from https://github.com/component/escape-html.git. This software contains the following license and notice below:
(The MIT License)
@@ -6929,11 +7220,89 @@
-----
-The following software may be included in this product: eslint-loader. A copy of the source code may be downloaded from https://github.com/webpack-contrib/eslint-loader.git. This software contains the following license and notice below:
+The following software may be included in this product: eslint-config-prettier. A copy of the source code may be downloaded from https://github.com/prettier/eslint-config-prettier.git. This software contains the following license and notice below:
The MIT License (MIT)
-Copyright (c) 2015 Maxime Thirouin
+Copyright (c) 2017, 2018, 2019 Simon Lydell and contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+-----
+
+The following software may be included in this product: eslint-import-resolver-webpack, eslint-module-utils, eslint-plugin-import. A copy of the source code may be downloaded from git+https://github.com/benmosher/eslint-plugin-import.git (eslint-import-resolver-webpack), git+https://github.com/benmosher/eslint-plugin-import.git (eslint-module-utils), https://github.com/benmosher/eslint-plugin-import (eslint-plugin-import). This software contains the following license and notice below:
+
+The MIT License (MIT)
+
+Copyright (c) 2015 Ben Mosher
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+-----
+
+The following software may be included in this product: eslint-plugin-es, eslint-utils, regexpp. A copy of the source code may be downloaded from git+https://github.com/mysticatea/eslint-plugin-es.git (eslint-plugin-es), git+https://github.com/mysticatea/eslint-utils.git (eslint-utils), git+https://github.com/mysticatea/regexpp.git (regexpp). This software contains the following license and notice below:
+
+MIT License
+
+Copyright (c) 2018 Toru Nagashima
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+-----
+
+The following software may be included in this product: eslint-plugin-eslint-comments. A copy of the source code may be downloaded from git+https://github.com/mysticatea/eslint-plugin-eslint-comments.git. This software contains the following license and notice below:
+
+MIT License
+
+Copyright (c) 2016 Toru Nagashima
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
@@ -6984,11 +7353,24 @@
-----
-The following software may be included in this product: eslint-plugin-import. A copy of the source code may be downloaded from https://github.com/benmosher/eslint-plugin-import. This software contains the following license and notice below:
+The following software may be included in this product: eslint-plugin-jsx-a11y, jsx-ast-utils. A copy of the source code may be downloaded from https://github.com/evcohen/eslint-plugin-jsx-a11y (eslint-plugin-jsx-a11y), https://github.com/evcohen/jsx-ast-utils (jsx-ast-utils). This software contains the following license and notice below:
+
+The MIT License (MIT)
+Copyright (c) 2016 Ethan Cohen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+-----
+
+The following software may be included in this product: eslint-plugin-no-use-extend-native, is-get-set-prop, is-js-type, is-obj-prop, is-proto-prop. A copy of the source code may be downloaded from https://github.com/dustinspecker/eslint-plugin-no-use-extend-native (eslint-plugin-no-use-extend-native), https://github.com/dustinspecker/is-get-set-prop.git (is-get-set-prop), https://github.com/dustinspecker/is-js-type.git (is-js-type), https://github.com/dustinspecker/is-obj-prop.git (is-obj-prop), https://github.com/dustinspecker/is-proto-prop.git (is-proto-prop). This software contains the following license and notice below:
The MIT License (MIT)
-Copyright (c) 2015 Ben Mosher
+Copyright (c) 2015 Dustin Specker
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
@@ -7010,16 +7392,58 @@
-----
-The following software may be included in this product: eslint-plugin-jsx-a11y, jsx-ast-utils. A copy of the source code may be downloaded from https://github.com/evcohen/eslint-plugin-jsx-a11y (eslint-plugin-jsx-a11y), https://github.com/evcohen/jsx-ast-utils (jsx-ast-utils). This software contains the following license and notice below:
+The following software may be included in this product: eslint-plugin-node, npm-run-all. A copy of the source code may be downloaded from git+https://github.com/mysticatea/eslint-plugin-node.git (eslint-plugin-node), https://github.com/mysticatea/npm-run-all.git (npm-run-all). This software contains the following license and notice below:
The MIT License (MIT)
-Copyright (c) 2016 Ethan Cohen
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+Copyright (c) 2015 Toru Nagashima
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+-----
+
+The following software may be included in this product: eslint-plugin-prettier, prettier-linter-helpers. A copy of the source code may be downloaded from git+https://github.com/prettier/eslint-plugin-prettier.git (eslint-plugin-prettier), git+https://github.com/prettier/prettier-linter-helpers.git (prettier-linter-helpers). This software contains the following license and notice below:
+
+# The MIT License (MIT)
+
+Copyright © 2017 Andres Suarez and Teddy Katz
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the “Software”), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
-----
@@ -7076,60 +7500,6 @@
-----
-The following software may be included in this product: eslint-scope. A copy of the source code may be downloaded from https://github.com/eslint/eslint-scope.git. This software contains the following license and notice below:
-
-eslint-scope
-Copyright JS Foundation and other contributors, https://js.foundation
-Copyright (C) 2012-2013 Yusuke Suzuki (twitter: @Constellation) and other contributors.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
-DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
-THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
------
-
-The following software may be included in this product: eslint-utils, regexpp. A copy of the source code may be downloaded from git+https://github.com/mysticatea/eslint-utils.git (eslint-utils), git+https://github.com/mysticatea/regexpp.git (regexpp). This software contains the following license and notice below:
-
-MIT License
-
-Copyright (c) 2018 Toru Nagashima
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
------
-
The following software may be included in this product: eslint-visitor-keys. A copy of the source code may be downloaded from https://github.com/eslint/eslint-visitor-keys.git. This software contains the following license and notice below:
Apache License
@@ -7915,35 +8285,29 @@
-----
-The following software may be included in this product: filesize. A copy of the source code may be downloaded from git://github.com/avoidwork/filesize.js.git. This software contains the following license and notice below:
+The following software may be included in this product: fill-range, is-number, merge-deep, micromatch. A copy of the source code may be downloaded from https://github.com/jonschlinkert/fill-range.git (fill-range), https://github.com/jonschlinkert/is-number.git (is-number), https://github.com/jonschlinkert/merge-deep.git (merge-deep), https://github.com/micromatch/micromatch.git (micromatch). This software contains the following license and notice below:
-Copyright (c) 2018, Jason Mulligan
-All rights reserved.
+The MIT License (MIT)
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
+Copyright (c) 2014-present, Jon Schlinkert.
-* Redistributions of source code must retain the above copyright notice, this
- list of conditions and the following disclaimer.
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
-* Redistributions in binary form must reproduce the above copyright notice,
- this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
-* Neither the name of filesize nor the names of its
- contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
-----
@@ -7974,7 +8338,7 @@
-----
-The following software may be included in this product: find-cache-dir. A copy of the source code may be downloaded from https://github.com/avajs/find-cache-dir.git. This software contains the following license and notice below:
+The following software may be included in this product: find-cache-dir, supports-hyperlinks. A copy of the source code may be downloaded from https://github.com/avajs/find-cache-dir.git (find-cache-dir), https://github.com/jamestalmage/supports-hyperlinks.git (supports-hyperlinks). This software contains the following license and notice below:
MIT License
@@ -7988,29 +8352,15 @@
-----
-The following software may be included in this product: find-cache-dir, node-modules-regexp, normalize-range. A copy of the source code may be downloaded from https://github.com/jamestalmage/find-cache-dir.git (find-cache-dir), https://github.com/jamestalmage/node-modules-regexp.git (node-modules-regexp), https://github.com/jamestalmage/normalize-range.git (normalize-range). This software contains the following license and notice below:
+The following software may be included in this product: find-root. A copy of the source code may be downloaded from git@github.com:js-n/find-root.git. This software contains the following license and notice below:
-The MIT License (MIT)
+Copyright © 2017 jsdnxx
-Copyright (c) James Talmage <james@talmage.io> (github.com/jamestalmage)
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-----
@@ -8339,7 +8689,34 @@
MIT License
-----------
-Copyright (C) 2010-2019 by Philipp Dunkel, Ben Noordhuis, Elan Shankar
+Copyright (C) 2010-2019 by Philipp Dunkel, Ben Noordhuis, Elan Shankar, Paul Miller
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+-----
+
+The following software may be included in this product: fsevents. A copy of the source code may be downloaded from https://github.com/fsevents/fsevents.git. This software contains the following license and notice below:
+
+MIT License
+-----------
+
+Copyright (C) 2010-2020 by Philipp Dunkel, Ben Noordhuis, Elan Shankar, Paul Miller
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
@@ -8411,6 +8788,18 @@
-----
+The following software may be included in this product: gensync. This software contains the following license and notice below:
+
+Copyright 2018 Logan Smyth <loganfsmyth@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+-----
+
The following software may be included in this product: get-caller-file. A copy of the source code may be downloaded from git+https://github.com/stefanpenner/get-caller-file.git. This software contains the following license and notice below:
ISC License (ISC)
@@ -8440,6 +8829,32 @@
-----
+The following software may be included in this product: get-set-props, obj-props. A copy of the source code may be downloaded from https://github.com/dustinspecker/get-set-props.git (get-set-props), https://github.com/dustinspecker/obj-props.git (obj-props). This software contains the following license and notice below:
+
+The MIT License (MIT)
+
+Copyright (c) Sindre Sorhus <sindresorhus@gmail.com> (sindresorhus.com), Dustin Specker <DustinSpecker@DustinSpecker.com> (github.com/dustinspecker)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+-----
+
The following software may be included in this product: getpass, http-signature, sshpk. A copy of the source code may be downloaded from https://github.com/arekinath/node-getpass.git (getpass), git://github.com/joyent/node-http-signature.git (http-signature), git+https://github.com/joyent/node-sshpk.git (sshpk). This software contains the following license and notice below:
Copyright Joyent, Inc. All rights reserved.
@@ -8489,6 +8904,26 @@
-----
+The following software may be included in this product: glob-parent. A copy of the source code may be downloaded from https://github.com/gulpjs/glob-parent.git. This software contains the following license and notice below:
+
+The ISC License
+
+Copyright (c) 2015, 2019 Elan Shanker
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
+IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+-----
+
The following software may be included in this product: glob-parent. A copy of the source code may be downloaded from https://github.com/es128/glob-parent. This software contains the following license and notice below:
The ISC License
@@ -8509,7 +8944,7 @@
-----
-The following software may be included in this product: global-modules, global-prefix, repeat-element, use. A copy of the source code may be downloaded from https://github.com/jonschlinkert/global-modules.git (global-modules), https://github.com/jonschlinkert/global-prefix.git (global-prefix), https://github.com/jonschlinkert/repeat-element.git (repeat-element), https://github.com/jonschlinkert/use.git (use). This software contains the following license and notice below:
+The following software may be included in this product: global-modules, global-prefix, repeat-element, shallow-clone, to-regex-range, use. A copy of the source code may be downloaded from https://github.com/jonschlinkert/global-modules.git (global-modules), https://github.com/jonschlinkert/global-prefix.git (global-prefix), https://github.com/jonschlinkert/repeat-element.git (repeat-element), https://github.com/jonschlinkert/shallow-clone.git (shallow-clone), https://github.com/micromatch/to-regex-range.git (to-regex-range), https://github.com/jonschlinkert/use.git (use). This software contains the following license and notice below:
The MIT License (MIT)
@@ -8867,9 +9302,9 @@
-----
-The following software may be included in this product: html-minifier. A copy of the source code may be downloaded from git+https://github.com/kangax/html-minifier.git. This software contains the following license and notice below:
+The following software may be included in this product: html-minifier-terser. A copy of the source code may be downloaded from git+https://github.com/DanielRuf/html-minifier-terser.git. This software contains the following license and notice below:
-Copyright (c) 2010-2018 Juriy "kangax" Zaytsev
+Copyright (c) 2010-2019 Juriy "kangax" Zaytsev
Permission is hereby granted, free of charge, to any person
obtaining a copy of this software and associated documentation
@@ -9382,7 +9817,7 @@
-----
-The following software may be included in this product: infer-owner. A copy of the source code may be downloaded from https://github.com/npm/infer-owner. This software contains the following license and notice below:
+The following software may be included in this product: infer-owner, minipass. A copy of the source code may be downloaded from https://github.com/npm/infer-owner (infer-owner), git+https://github.com/isaacs/minipass.git (minipass). This software contains the following license and notice below:
The ISC License
@@ -9469,6 +9904,59 @@
-----
+The following software may be included in this product: internal-slot, side-channel. A copy of the source code may be downloaded from git+https://github.com/ljharb/internal-slot.git (internal-slot), git+https://github.com/ljharb/side-channel.git (side-channel). This software contains the following license and notice below:
+
+MIT License
+
+Copyright (c) 2019 Jordan Harband
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+-----
+
+The following software may be included in this product: interpret. A copy of the source code may be downloaded from https://github.com/gulpjs/interpret.git. This software contains the following license and notice below:
+
+Copyright (c) 2014-2018 Tyler Kellen <tyler@sleekcode.net>, Blaine Bublitz <blaine.bublitz@gmail.com>, and Eric Schoffstall <yo@contra.io>
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+-----
+
The following software may be included in this product: ipaddr.js. A copy of the source code may be downloaded from git://github.com/whitequark/ipaddr.js. This software contains the following license and notice below:
Copyright (C) 2011-2017 whitequark <whitequark@whitequark.org>
@@ -9493,6 +9981,33 @@
-----
+The following software may be included in this product: is-absolute. A copy of the source code may be downloaded from https://github.com/jonschlinkert/is-absolute.git. This software contains the following license and notice below:
+
+The MIT License (MIT)
+
+Copyright (c) 2014-2017, Jon Schlinkert.
+Copyright (c) 2009-2014, TJ Holowaychuk
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+-----
+
The following software may be included in this product: is-arguments, is-regex, object-is. A copy of the source code may be downloaded from git://github.com/ljharb/is-arguments.git (is-arguments), git://github.com/ljharb/is-regex.git (is-regex), git://github.com/es-shims/object-is.git (object-is). This software contains the following license and notice below:
The MIT License (MIT)
@@ -9518,7 +10033,7 @@
-----
-The following software may be included in this product: is-buffer, safe-buffer. A copy of the source code may be downloaded from git://github.com/feross/is-buffer.git (is-buffer), git://github.com/feross/safe-buffer.git (safe-buffer). This software contains the following license and notice below:
+The following software may be included in this product: is-buffer, safe-buffer, typedarray-to-buffer. A copy of the source code may be downloaded from git://github.com/feross/is-buffer.git (is-buffer), git://github.com/feross/safe-buffer.git (safe-buffer), git://github.com/feross/typedarray-to-buffer.git (typedarray-to-buffer). This software contains the following license and notice below:
The MIT License (MIT)
@@ -9570,7 +10085,7 @@
-----
-The following software may be included in this product: is-descriptor, is-extendable, shallow-clone. A copy of the source code may be downloaded from https://github.com/jonschlinkert/is-descriptor.git (is-descriptor), https://github.com/jonschlinkert/is-extendable.git (is-extendable), https://github.com/jonschlinkert/shallow-clone.git (shallow-clone). This software contains the following license and notice below:
+The following software may be included in this product: is-descriptor, is-extendable, is-unc-path. A copy of the source code may be downloaded from https://github.com/jonschlinkert/is-descriptor.git (is-descriptor), https://github.com/jonschlinkert/is-extendable.git (is-extendable), https://github.com/jonschlinkert/is-unc-path.git (is-unc-path). This software contains the following license and notice below:
The MIT License (MIT)
@@ -9596,6 +10111,56 @@
-----
+The following software may be included in this product: is-error. A copy of the source code may be downloaded from git://github.com/mk-pmb/is-error-js.git. This software contains the following license and notice below:
+
+Copyright (c) 2015 is-error.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+-----
+
+The following software may be included in this product: is-negated-glob. A copy of the source code may be downloaded from https://github.com/jonschlinkert/is-negated-glob.git. This software contains the following license and notice below:
+
+The MIT License (MIT)
+
+Copyright (c) 2016 Jon Schlinkert
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+-----
+
The following software may be included in this product: is-resolvable. A copy of the source code may be downloaded from https://github.com/shinnn/is-resolvable.git. This software contains the following license and notice below:
ISC License (ISC)
@@ -9811,32 +10376,6 @@
-----
-The following software may be included in this product: js-levenshtein. A copy of the source code may be downloaded from https://github.com/gustf/js-levenshtein.git. This software contains the following license and notice below:
-
-MIT License
-
-Copyright (c) 2017 Gustaf Andersson
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
------
-
The following software may be included in this product: js-tokens. A copy of the source code may be downloaded from https://github.com/lydell/js-tokens.git. This software contains the following license and notice below:
The MIT License (MIT)
@@ -10448,6 +10987,32 @@
-----
+The following software may be included in this product: levenary. A copy of the source code may be downloaded from https://github.com/tanhauhau/levenary.git. This software contains the following license and notice below:
+
+The MIT License (MIT)
+
+Copyright (c) 2019 Tan Li Hau
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+-----
+
The following software may be included in this product: levn, optionator, prelude-ls, type-check. A copy of the source code may be downloaded from git://github.com/gkz/levn.git (levn), git://github.com/gkz/optionator.git (optionator), git://github.com/gkz/prelude-ls.git (prelude-ls), git://github.com/gkz/type-check.git (type-check). This software contains the following license and notice below:
Copyright (c) George Zahariev
@@ -10475,7 +11040,33 @@
-----
-The following software may be included in this product: loader-fs-cache. This software contains the following license and notice below:
+The following software may be included in this product: lines-and-columns. A copy of the source code may be downloaded from https://github.com/eventualbuddha/lines-and-columns.git. This software contains the following license and notice below:
+
+The MIT License (MIT)
+
+Copyright (c) 2015 Brian Donovan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+-----
+
+The following software may be included in this product: loader-fs-cache. A copy of the source code may be downloaded from https://github.com/viankakrisna/loader-fs-cache.git. This software contains the following license and notice below:
Copyright (c) 2014-2016 Ade Viankakrisna Fadlil <viankakrisna@gmail.com>
@@ -10632,7 +11223,7 @@
-----
-The following software may be included in this product: lodash.debounce, lodash.flow, lodash.memoize, lodash.sortby, lodash.tail, lodash.throttle, lodash.unescape, lodash.uniq. A copy of the source code may be downloaded from https://github.com/lodash/lodash.git (lodash.debounce), https://github.com/lodash/lodash.git (lodash.flow), https://github.com/lodash/lodash.git (lodash.memoize), https://github.com/lodash/lodash.git (lodash.sortby), https://github.com/lodash/lodash.git (lodash.tail), https://github.com/lodash/lodash.git (lodash.throttle), https://github.com/lodash/lodash.git (lodash.unescape), https://github.com/lodash/lodash.git (lodash.uniq). This software contains the following license and notice below:
+The following software may be included in this product: lodash.debounce, lodash.flow, lodash.get, lodash.memoize, lodash.sortby, lodash.throttle, lodash.uniq, lodash.zip. A copy of the source code may be downloaded from https://github.com/lodash/lodash.git (lodash.debounce), https://github.com/lodash/lodash.git (lodash.flow), https://github.com/lodash/lodash.git (lodash.get), https://github.com/lodash/lodash.git (lodash.memoize), https://github.com/lodash/lodash.git (lodash.sortby), https://github.com/lodash/lodash.git (lodash.throttle), https://github.com/lodash/lodash.git (lodash.uniq), https://github.com/lodash/lodash.git (lodash.zip). This software contains the following license and notice below:
Copyright jQuery Foundation and other contributors <https://jquery.org/>
@@ -10768,20 +11359,6 @@
-----
-The following software may be included in this product: map-age-cleaner. A copy of the source code may be downloaded from https://github.com/SamVerschueren/map-age-cleaner.git. This software contains the following license and notice below:
-
-MIT License
-
-Copyright (c) Sam Verschueren <sam.verschueren@gmail.com> (github.com/SamVerschueren)
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
------
-
The following software may be included in this product: map-visit, to-regex-range, union-value. A copy of the source code may be downloaded from https://github.com/jonschlinkert/map-visit.git (map-visit), https://github.com/micromatch/to-regex-range.git (to-regex-range), https://github.com/jonschlinkert/union-value.git (union-value). This software contains the following license and notice below:
The MIT License (MIT)
@@ -10808,7 +11385,7 @@
-----
-The following software may be included in this product: mdn-data. A copy of the source code may be downloaded from https://github.com/mdn/data.git. This software contains the following license and notice below:
+The following software may be included in this product: mdn-data, micro-spelling-correcter. A copy of the source code may be downloaded from https://github.com/mdn/data.git (mdn-data). This software contains the following license and notice below:
CC0 1.0 Universal
@@ -10980,32 +11557,6 @@
-----
-The following software may be included in this product: merge-deep. A copy of the source code may be downloaded from https://github.com/jonschlinkert/merge-deep.git. This software contains the following license and notice below:
-
-The MIT License (MIT)
-
-Copyright (c) 2014-present, Jon Schlinkert.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-
------
-
The following software may be included in this product: merge-descriptors. A copy of the source code may be downloaded from https://github.com/component/merge-descriptors.git. This software contains the following license and notice below:
(The MIT License)
@@ -11238,6 +11789,32 @@
-----
+The following software may be included in this product: minimist-options. A copy of the source code may be downloaded from https://github.com/vadimdemedes/minimist-options.git. This software contains the following license and notice below:
+
+The MIT License (MIT)
+
+Copyright (c) Vadim Demedes <vdemedes@gmail.com> (vadimdemedes.com)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+-----
+
The following software may be included in this product: mississippi. A copy of the source code may be downloaded from git+https://github.com/maxogden/mississippi.git. This software contains the following license and notice below:
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
@@ -11506,6 +12083,32 @@
-----
+The following software may be included in this product: next-tick. A copy of the source code may be downloaded from git://github.com/medikoo/next-tick.git. This software contains the following license and notice below:
+
+The MIT License
+
+Copyright (C) 2012-2016 Mariusz Nowak
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+-----
+
The following software may be included in this product: nice-try. A copy of the source code may be downloaded from https://github.com/electerious/nice-try.git. This software contains the following license and notice below:
The MIT License (MIT)
@@ -12031,32 +12634,6 @@
-----
-The following software may be included in this product: npm-run-all. A copy of the source code may be downloaded from https://github.com/mysticatea/npm-run-all.git. This software contains the following license and notice below:
-
-The MIT License (MIT)
-
-Copyright (c) 2015 Toru Nagashima
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
------
-
The following software may be included in this product: num2fraction, postcss-media-minmax. A copy of the source code may be downloaded from git@github.com:yisibl/num2fraction.git (num2fraction), https://github.com/postcss/postcss-media-minmax.git (postcss-media-minmax). This software contains the following license and notice below:
The MIT License (MIT)
@@ -12214,6 +12791,32 @@
-----
+The following software may be included in this product: object-path. A copy of the source code may be downloaded from git://github.com/mariocasciaro/object-path.git. This software contains the following license and notice below:
+
+The MIT License (MIT)
+
+Copyright (c) 2015 Mario Casciaro
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+-----
+
The following software may be included in this product: object.assign. A copy of the source code may be downloaded from git://github.com/ljharb/object.assign.git. This software contains the following license and notice below:
The MIT License (MIT)
@@ -12660,6 +13263,32 @@
-----
+The following software may be included in this product: picomatch. A copy of the source code may be downloaded from https://github.com/micromatch/picomatch.git. This software contains the following license and notice below:
+
+The MIT License (MIT)
+
+Copyright (c) 2017-present, Jon Schlinkert.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+-----
+
The following software may be included in this product: pidtree. A copy of the source code may be downloaded from https://github.com/simonepri/pidtree.git. This software contains the following license and notice below:
MIT License
@@ -12790,7 +13419,7 @@
-----
-The following software may be included in this product: portfinder. A copy of the source code may be downloaded from git@github.com:indexzero/node-portfinder.git. This software contains the following license and notice below:
+The following software may be included in this product: portfinder. A copy of the source code may be downloaded from git@github.com:http-party/node-portfinder.git. This software contains the following license and notice below:
node-portfinder
@@ -13253,6 +13882,18 @@
-----
+The following software may be included in this product: prettier. A copy of the source code may be downloaded from https://github.com/prettier/prettier.git. This software contains the following license and notice below:
+
+Copyright © James Long and contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+-----
+
The following software may be included in this product: private. A copy of the source code may be downloaded from git://github.com/benjamn/private.git. This software contains the following license and notice below:
Copyright (c) 2014 Ben Newman <bn@cs.stanford.edu>
@@ -13977,6 +14618,32 @@
-----
+The following software may be included in this product: readdirp. A copy of the source code may be downloaded from git://github.com/paulmillr/readdirp.git. This software contains the following license and notice below:
+
+MIT License
+
+Copyright (c) 2012-2019 Thorsten Lorenz, Paul Miller (https://paulmillr.com)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+-----
+
The following software may be included in this product: realpath-native. A copy of the source code may be downloaded from https://github.com/SimenB/realpath-native.git. This software contains the following license and notice below:
MIT License
@@ -14056,6 +14723,58 @@
-----
+The following software may be included in this product: regex-parser. A copy of the source code may be downloaded from git@github.com:IonicaBizau/regex-parser.js.git. This software contains the following license and notice below:
+
+The MIT License (MIT)
+
+Copyright (c) 2014-19 Ionică Bizău <bizauionica@gmail.com> (https://ionicabizau.net)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+-----
+
+The following software may be included in this product: regexp-tree. A copy of the source code may be downloaded from https://github.com/DmitrySoshnikov/regexp-tree.git. This software contains the following license and notice below:
+
+MIT License
+
+Copyright (c) 2017 Dmitry Soshnikov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+-----
+
The following software may be included in this product: regexp.prototype.flags. A copy of the source code may be downloaded from git://github.com/es-shims/RegExp.prototype.flags.git. This software contains the following license and notice below:
The MIT License (MIT)
@@ -14286,6 +15005,33 @@
-----
+The following software may be included in this product: reserved-words. A copy of the source code may be downloaded from https://github.com/zxqfox/reserved-words.git. This software contains the following license and notice below:
+
+The MIT License (MIT)
+
+Copyright 2015 Alexej Yaroshevich and other contributors
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+-----
+
The following software may be included in this product: resize-observer-polyfill. A copy of the source code may be downloaded from https://github.com/que-etc/resize-observer-polyfill.git. This software contains the following license and notice below:
The MIT License (MIT)
@@ -14438,6 +15184,32 @@
-----
+The following software may be included in this product: retry. A copy of the source code may be downloaded from git://github.com/tim-kos/node-retry.git. This software contains the following license and notice below:
+
+Copyright (c) 2011:
+Tim Koschützki (tim@debuggable.com)
+Felix Geisendörfer (felix@debuggable.com)
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+
+-----
+
The following software may be included in this product: ripemd160. A copy of the source code may be downloaded from https://github.com/crypto-browserify/ripemd160. This software contains the following license and notice below:
The MIT License (MIT)
@@ -14720,6 +15492,34 @@
-----
+The following software may be included in this product: safe-regex. A copy of the source code may be downloaded from git://github.com/davisjam/safe-regex.git. This software contains the following license and notice below:
+
+Copyright 2019-present is held by the authors of the safe-regex module.
+
+This software is released under the MIT license:
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+Original author: James Halliday @substack
+Maintainer: James C. (Jamie) Davis @davisjam
+
+-----
+
The following software may be included in this product: safer-buffer. A copy of the source code may be downloaded from git+https://github.com/ChALkeR/safer-buffer.git. This software contains the following license and notice below:
MIT License
@@ -14750,22 +15550,24 @@
Copyright JS Foundation and other contributors
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software is furnished to do so,
-subject to the following conditions:
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
-IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-----
@@ -15367,32 +16169,6 @@
-----
-The following software may be included in this product: sockjs-client. A copy of the source code may be downloaded from https://github.com/sockjs/sockjs-client.git. This software contains the following license and notice below:
-
-The MIT License (MIT)
-
-Copyright (c) 2011-2012 VMware, Inc.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-
------
-
The following software may be included in this product: source-list-map. A copy of the source code may be downloaded from https://github.com/webpack/source-list-map.git. This software contains the following license and notice below:
Copyright 2017 JS Foundation
@@ -15702,6 +16478,32 @@
-----
+The following software may be included in this product: strict-uri-encode. A copy of the source code may be downloaded from https://github.com/kevva/strict-uri-encode.git. This software contains the following license and notice below:
+
+The MIT License (MIT)
+
+Copyright (c) Kevin Mårtensson <kevinmartensson@gmail.com> (github.com/kevva)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+-----
+
The following software may be included in this product: string_decoder. A copy of the source code may be downloaded from git://github.com/nodejs/string_decoder.git. This software contains the following license and notice below:
Node.js is licensed for use as follows:
@@ -15948,7 +16750,7 @@
-----
-The following software may be included in this product: terser, uglify-js. A copy of the source code may be downloaded from https://github.com/terser/terser (terser), https://github.com/mishoo/UglifyJS2.git (uglify-js). This software contains the following license and notice below:
+The following software may be included in this product: terser. A copy of the source code may be downloaded from https://github.com/terser/terser. This software contains the following license and notice below:
UglifyJS is released under the BSD license:
@@ -16373,6 +17175,23 @@
-----
+The following software may be included in this product: tslib. A copy of the source code may be downloaded from https://github.com/Microsoft/tslib.git. This software contains the following license and notice below:
+
+Copyright (c) Microsoft Corporation.
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.
+
+-----
+
The following software may be included in this product: tsutils. A copy of the source code may be downloaded from https://github.com/ajafff/tsutils. This software contains the following license and notice below:
The MIT License (MIT)
@@ -16428,6 +17247,26 @@
-----
+The following software may be included in this product: type. A copy of the source code may be downloaded from https://github.com/medikoo/type.git. This software contains the following license and notice below:
+
+ISC License
+
+Copyright (c) 2019, Mariusz Nowak, @medikoo, medikoo.com
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.
+
+-----
+
The following software may be included in this product: typedarray. A copy of the source code may be downloaded from git://github.com/substack/typedarray.git. This software contains the following license and notice below:
/*
@@ -16837,6 +17676,32 @@
-----
+The following software may be included in this product: v8-compile-cache. A copy of the source code may be downloaded from https://github.com/zertosh/v8-compile-cache.git. This software contains the following license and notice below:
+
+The MIT License (MIT)
+
+Copyright (c) 2019 Andres Suarez
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+-----
+
The following software may be included in this product: vendors. A copy of the source code may be downloaded from https://github.com/wooorm/vendors.git. This software contains the following license and notice below:
(The MIT License)
@@ -17154,7 +18019,7 @@
-----
-The following software may be included in this product: write-file-atomic. A copy of the source code may be downloaded from git@github.com:iarna/write-file-atomic.git. This software contains the following license and notice below:
+The following software may be included in this product: write-file-atomic. A copy of the source code may be downloaded from git://github.com/npm/write-file-atomic.git. This software contains the following license and notice below:
Copyright (c) 2015, Rebecca Turner
@@ -17398,7 +18263,7 @@
The MIT License
-Copyright (c) 2007-2017 Steven Levithan <http://xregexp.com/>
+Copyright (c) 2007-present Steven Levithan <http://xregexp.com/>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
@@ -17463,6 +18328,24 @@
-----
+The following software may be included in this product: yaml. A copy of the source code may be downloaded from https://github.com/eemeli/yaml.git. This software contains the following license and notice below:
+
+Copyright 2018 Eemeli Aro <eemeli@gmail.com>
+
+Permission to use, copy, modify, and/or distribute this software for any purpose
+with or without fee is hereby granted, provided that the above copyright notice
+and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
+THIS SOFTWARE.
+
+-----
+
The following software may be included in this product: yargs. A copy of the source code may be downloaded from https://github.com/yargs/yargs.git. This software contains the following license and notice below:
Copyright 2010 James Halliday (mail@substack.net)
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json
index ab807df..77d3a69 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json
@@ -1,16 +1,16 @@
{
"clusterState": {
- "totalDatanodes": 10,
- "healthyDatanodes": 9,
- "pipelines": 3,
+ "totalDatanodes": 24,
+ "healthyDatanodes": 24,
+ "pipelines": 32,
"storageReport": {
- "capacity": 1099511627778,
- "used": 681826058240,
- "remaining": 391915765760
+ "capacity": 32985348833280,
+ "used": 15942918602752,
+ "remaining": 12094627905536
},
- "containers": 54,
+ "containers": 3230,
"volumes": 5,
- "buckets": 100,
+ "buckets": 156,
"keys": 253000
},
"datanodes": {
@@ -29,15 +29,18 @@
{
"pipelineID": "02e3d908-ff01-4ce6-ad75-f3ec79bcc71a",
"replicationType": "RATIS",
- "replicationFactor": 3
+ "replicationFactor": 3,
+ "leaderNode": "localhost1.storage.enterprise.com"
},
{
"pipelineID": "09d3a478-ff01-4ce6-ad75-f3ec79bcc71a",
"replicationType": "RATIS",
- "replicationFactor": 1
+ "replicationFactor": 1,
+ "leaderNode": "localhost1.storage.enterprise.com"
}
],
- "containers": 80
+ "containers": 80,
+ "leaderCount": 2
},
{
"hostname": "localhost2.storage.enterprise.com",
@@ -52,15 +55,18 @@
{
"pipelineID": "02e3d908-ff01-4ce6-ad75-f3ec79bcc71a",
"replicationType": "RATIS",
- "replicationFactor": 3
+ "replicationFactor": 3,
+ "leaderNode": "localhost1.storage.enterprise.com"
},
{
"pipelineID": "05e3d908-ff01-4ce6-ad75-f3ec79bcc7982",
"replicationType": "RATIS",
- "replicationFactor": 1
+ "replicationFactor": 1,
+ "leaderNode": "localhost2.storage.enterprise.com"
}
],
- "containers": 8192
+ "containers": 8192,
+ "leaderCount": 1
},
{
"hostname": "localhost3.storage.enterprise.com",
@@ -75,20 +81,24 @@
{
"pipelineID": "02e3d908-ff01-4ce6-ad75-f3ec79bcc71a",
"replicationType": "RATIS",
- "replicationFactor": 3
+ "replicationFactor": 3,
+ "leaderNode": "localhost1.storage.enterprise.com"
},
{
"pipelineID": "05e3d908-ff01-4ce6-ad75-f3ec79bcc7982",
"replicationType": "RATIS",
- "replicationFactor": 1
+ "replicationFactor": 1,
+ "leaderNode": "localhost3.storage.enterprise.com"
},
{
"pipelineID": "02e3d908-ff01-4ce6-ad75-f3ec79bcc71a",
"replicationType": "STAND_ALONE",
- "replicationFactor": 1
+ "replicationFactor": 1,
+ "leaderNode": "localhost3.storage.enterprise.com"
}
],
- "containers": 43
+ "containers": 43,
+ "leaderCount": 2
},
{
"hostname": "localhost4.storage.enterprise.com",
@@ -100,7 +110,8 @@
"remaining": 110737488355328
},
"pipelines": [],
- "containers": 0
+ "containers": 0,
+ "leaderCount": 0
},
{
"hostname": "localhost5.storage.enterprise.com",
@@ -115,15 +126,18 @@
{
"pipelineID": "02e3d908-ff01-4ce6-ad75-f3ec79bcc71a",
"replicationType": "RATIS",
- "replicationFactor": 3
+ "replicationFactor": 3,
+ "leaderNode": "localhost5.storage.enterprise.com"
},
{
"pipelineID": "05e3d908-ff01-4ce6-ad75-f3ec79bcc7982",
"replicationType": "RATIS",
- "replicationFactor": 1
+ "replicationFactor": 1,
+ "leaderNode": "localhost5.storage.enterprise.com"
}
],
- "containers": 643
+ "containers": 643,
+ "leaderCount": 2
},
{
"hostname": "localhost6.storage.enterprise.com",
@@ -138,15 +152,18 @@
{
"pipelineID": "02e3d908-ff01-4ce6-ad75-f3ec79bcc71a",
"replicationType": "RATIS",
- "replicationFactor": 3
+ "replicationFactor": 3,
+ "leaderNode": "localhost5.storage.enterprise.com"
},
{
"pipelineID": "05e3d908-ff01-4ce6-ad75-f3ec79bcc7982",
"replicationType": "RATIS",
- "replicationFactor": 1
+ "replicationFactor": 1,
+ "leaderNode": "localhost6.storage.enterprise.com"
}
],
- "containers": 5
+ "containers": 5,
+ "leaderCount": 1
},
{
"hostname": "localhost7.storage.enterprise.com",
@@ -161,20 +178,24 @@
{
"pipelineID": "02e3d908-ff01-4ce6-ad75-f3ec79bcc71a",
"replicationType": "RATIS",
- "replicationFactor": 3
+ "replicationFactor": 3,
+ "leaderNode": "localhost5.storage.enterprise.com"
},
{
"pipelineID": "05e3d908-ff01-4ce6-ad75-f3ec79bcc7982",
"replicationType": "RATIS",
- "replicationFactor": 1
+ "replicationFactor": 1,
+ "leaderNode": "localhost7.storage.enterprise.com"
},
{
"pipelineID": "05e3d908-ff01-4ce6-ad75-f3ec79bcc7982",
"replicationType": "STAND_ALONE",
- "replicationFactor": 1
+ "replicationFactor": 1,
+ "leaderNode": "localhost7.storage.enterprise.com"
}
],
- "containers": 64
+ "containers": 64,
+ "leaderCount": 2
},
{
"hostname": "localhost8.storage.enterprise.com",
@@ -189,15 +210,18 @@
{
"pipelineID": "02e3d908-ff01-4ce6-ad75-f3ec79bcc71a",
"replicationType": "RATIS",
- "replicationFactor": 3
+ "replicationFactor": 3,
+ "leaderNode": "localhost5.storage.enterprise.com"
},
{
"pipelineID": "05e3d908-ff01-4ce6-ad75-f3ec79bcc7982",
"replicationType": "RATIS",
- "replicationFactor": 1
+ "replicationFactor": 1,
+ "leaderNode": "localhost8.storage.enterprise.com"
}
],
- "containers": 21
+ "containers": 21,
+ "leaderCount": 1
},
{
"hostname": "localhost9.storage.enterprise.com",
@@ -212,15 +236,18 @@
{
"pipelineID": "02e3d908-ff01-4ce6-ad75-f3ec79bcc71a",
"replicationType": "RATIS",
- "replicationFactor": 3
+ "replicationFactor": 3,
+ "leaderNode": "localhost11.storage.enterprise.com"
},
{
"pipelineID": "05e3d908-ff01-4ce6-ad75-f3ec79bcc7982",
"replicationType": "RATIS",
- "replicationFactor": 1
+ "replicationFactor": 1,
+ "leaderNode": "localhost9.storage.enterprise.com"
}
],
- "containers": 897
+ "containers": 897,
+ "leaderCount": 1
},
{
"hostname": "localhost10.storage.enterprise.com",
@@ -235,20 +262,24 @@
{
"pipelineID": "02e3d908-ff01-4ce6-ad75-f3ec79bcc71a",
"replicationType": "RATIS",
- "replicationFactor": 3
+ "replicationFactor": 3,
+ "leaderNode": "localhost11.storage.enterprise.com"
},
{
"pipelineID": "05e3d908-ff01-4ce6-ad75-f3ec79bcc7982",
"replicationType": "RATIS",
- "replicationFactor": 1
+ "replicationFactor": 1,
+ "leaderNode": "localhost10.storage.enterprise.com"
},
{
"pipelineID": "01f2e105-ff01-4ce6-ad75-f3ec79bcc7982",
"replicationType": "STAND_ALONE",
- "replicationFactor": 1
+ "replicationFactor": 1,
+ "leaderNode": "localhost10.storage.enterprise.com"
}
],
- "containers": 6754
+ "containers": 6754,
+ "leaderCount": 2
},
{
"hostname": "localhost11.storage.enterprise.com",
@@ -263,15 +294,18 @@
{
"pipelineID": "02e3d908-ff01-4ce6-ad75-f3ec79bcc71a",
"replicationType": "RATIS",
- "replicationFactor": 3
+ "replicationFactor": 3,
+ "leaderNode": "localhost11.storage.enterprise.com"
},
{
"pipelineID": "05e3d908-ff01-4ce6-ad75-f3ec79bcc7982",
"replicationType": "RATIS",
- "replicationFactor": 1
+ "replicationFactor": 1,
+ "leaderNode": "localhost11.storage.enterprise.com"
}
],
- "containers": 78
+ "containers": 78,
+ "leaderCount": 2
},
{
"hostname": "localhost12.storage.enterprise.com",
@@ -286,15 +320,18 @@
{
"pipelineID": "02e3d908-ff01-4ce6-ad75-f3ec79bcc71a",
"replicationType": "RATIS",
- "replicationFactor": 3
+ "replicationFactor": 3,
+ "leaderNode": "localhost11.storage.enterprise.com"
},
{
"pipelineID": "05e3d908-ff01-4ce6-ad75-f3ec79bcc7982",
"replicationType": "RATIS",
- "replicationFactor": 1
+ "replicationFactor": 1,
+ "leaderNode": "localhost12.storage.enterprise.com"
}
],
- "containers": 543
+ "containers": 543,
+ "leaderCount": 1
}
]
},
@@ -355,26 +392,56 @@
"totalCount": 2,
"containers": [
{
- "id": 1,
- "keys": 3876,
- "datanodes": [
- "localhost1.storage.enterprise.com",
- "localhost3.storage.enterprise.com",
- "localhost5.storage.enterprise.com"
+ "containerID": 1,
+ "keys": 1235,
+ "replicas": [
+ {
+ "containerId": 1,
+ "datanodeHost": "localhost1.storage.enterprise.com",
+ "firstReportTimestamp": 1578491371100,
+ "lastReportTimestamp": 1578491371528
+ },
+ {
+ "containerId": 1,
+ "datanodeHost": "localhost3.storage.enterprise.com",
+ "firstReportTimestamp": 1578491370100,
+ "lastReportTimestamp": 1578491371328
+ },
+ {
+ "containerId": 1,
+ "datanodeHost": "localhost5.storage.enterprise.com",
+ "firstReportTimestamp": 1578491371200,
+ "lastReportTimestamp": 1578491371528
+ }
],
"missingSince": 1578491371528,
- "pipelineId": "05e3d908-ff01-4ce6-ad75-f3ec79bcc7982"
+ "pipelineID": "05e3d908-ff01-4ce6-ad75-f3ec79bcc7982"
},
{
- "id": 2,
- "keys": 5943,
- "datanodes": [
- "localhost1.storage.enterprise.com",
- "localhost3.storage.enterprise.com",
- "localhost5.storage.enterprise.com"
+ "containerID": 2,
+ "keys": 1356,
+ "replicas": [
+ {
+ "containerId": 1,
+ "datanodeHost": "localhost1.storage.enterprise.com",
+ "firstReportTimestamp": 1578491371100,
+ "lastReportTimestamp": 1578491371528
+ },
+ {
+ "containerId": 1,
+ "datanodeHost": "localhost3.storage.enterprise.com",
+ "firstReportTimestamp": 1578491370100,
+ "lastReportTimestamp": 1578491371328
+ },
+ {
+ "containerId": 1,
+ "datanodeHost": "localhost5.storage.enterprise.com",
+ "firstReportTimestamp": 1578491371200,
+ "lastReportTimestamp": 1578491371528
+ }
],
"missingSince": 1578491471528,
- "pipelineId": "04a5d908-ff01-4ce6-ad75-f3ec73dfc8a2"
+ "pipelineID": "04a5d908-ff01-4ce6-ad75-f3ec73dfc8a2"
}
]
},
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/routes.json b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/routes.json
index df5cec3..0e9f320 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/routes.json
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/routes.json
@@ -1,4 +1,5 @@
{
"/api/v1/*": "/$1",
- "/containers/:id/keys": "/keys"
+ "/containers/:id/keys": "/keys",
+ "/containers/missing": "/missingContainers"
}
\ No newline at end of file
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/config-overrides.js b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/config-overrides.js
index d29b530..c63cc97 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/config-overrides.js
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/config-overrides.js
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-const { override, fixBabelImports, addLessLoader} = require('customize-cra');
+const {override, fixBabelImports, addLessLoader} = require('customize-cra');
module.exports = override(
fixBabelImports('import', {
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/package.json b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/package.json
index decd77c..d9ac275 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/package.json
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/package.json
@@ -4,6 +4,7 @@
"private": true,
"dependencies": {
"@babel/core": "^7.0.0",
+ "@types/classnames": "^2.2.10",
"@types/jest": "24.0.12",
"@types/node": "11.13.9",
"@types/react": "16.8.15",
@@ -11,7 +12,7 @@
"@types/react-router-dom": "^4.3.3",
"antd": "^3.26.9",
"axios": "^0.19.0",
- "babel-jest": "24.7.1",
+ "babel-jest": "^24.9.0",
"babel-plugin-import": "^1.11.0",
"classnames": "^2.2.6",
"customize-cra": "^0.2.12",
@@ -24,8 +25,9 @@
"react": "^16.8.6",
"react-app-rewired": "^2.1.3",
"react-dom": "^16.8.6",
+ "react-router": "^5.1.2",
"react-router-dom": "^5.0.0",
- "react-scripts": "3.0.0",
+ "react-scripts": "^3.1.2",
"typescript": "3.4.5"
},
"scripts": {
@@ -35,7 +37,8 @@
"eject": "react-scripts eject",
"mock:api": "json-server --watch api/db.json --routes api/routes.json --port 9888",
"dev": "npm-run-all --parallel mock:api start",
- "lint": "eslint '**/*.tsx'"
+ "lint": "xo src/*",
+ "lint:fix": "xo --fix src/*"
},
"eslintConfig": {
"extends": "react-app"
@@ -53,8 +56,107 @@
]
},
"devDependencies": {
+ "@typescript-eslint/eslint-plugin": "^2.31.0",
+ "@typescript-eslint/parser": "^2.31.0",
+ "eslint": "^6.6.0",
+ "eslint-config-xo": "^0.29.1",
+ "eslint-config-xo-react": "^0.23.0",
+ "eslint-config-xo-typescript": "^0.28.0",
+ "eslint-plugin-import": "^2.20.2",
+ "eslint-plugin-react": "^7.19.0",
+ "eslint-plugin-unicorn": "^19.0.1",
"json-server": "^0.15.1",
- "npm-run-all": "^4.1.5"
+ "npm-run-all": "^4.1.5",
+ "xo": "^0.30.0"
+ },
+ "xo": {
+ "space": true,
+ "parser": "@typescript-eslint/parser",
+ "settings": {
+ "react": {
+ "pragma": "React",
+ "version": "16.8.6"
+ }
+ },
+ "rules": {
+ "jsx-quotes": [
+ 2,
+ "prefer-single"
+ ],
+ "import/no-unassigned-import": 0,
+ "@typescript-eslint/explicit-function-return-type": "off",
+ "@typescript-eslint/prefer-readonly-parameter-types": "off",
+ "@typescript-eslint/interface-name-prefix": [
+ 2,
+ "always"
+ ],
+ "import/no-extraneous-dependencies": [
+ "error",
+ {
+ "devDependencies": true,
+ "optionalDependencies": true,
+ "peerDependencies": true
+ }
+ ],
+ "camelcase": 0,
+ "react/state-in-constructor": 0,
+ "react/boolean-prop-naming": 0,
+ "promise/prefer-await-to-then": 0,
+ "react/require-default-props": 0,
+ "react/default-props-match-prop-types": 0,
+ "unicorn/prevent-abbreviations": 0,
+ "react/no-array-index-key": 0,
+ "no-return-assign": 0,
+ "indent": [
+ 2,
+ 2,
+ {
+ "SwitchCase": 1
+ }
+ ],
+ "unicorn/filename-case": [
+ "error",
+ {
+ "case": "camelCase",
+ "ignore": [
+ "^react-app-env"
+ ]
+ }
+ ],
+ "import/extensions": [
+ "error",
+ "ignorePackages",
+ {
+ "js": "never",
+ "ts": "never",
+ "tsx": "never"
+ }
+ ],
+ "no-unused-vars": [
+ 2,
+ {
+ "argsIgnorePattern": "^_\\w*",
+ "varsIgnorePattern": "^_\\w*"
+ }
+ ],
+ "eslint-comments/disable-enable-pair": [
+ 2,
+ {
+ "allowWholeFile": true
+ }
+ ]
+ },
+ "env": [
+ "node",
+ "es6",
+ "mocha",
+ "jest",
+ "browser"
+ ],
+ "extends": [
+ "xo-react/space",
+ "plugin:@typescript-eslint/recommended"
+ ]
},
"proxy": "http://localhost:9888"
}
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/App.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/App.tsx
deleted file mode 100644
index 5e289b0..0000000
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/App.tsx
+++ /dev/null
@@ -1,88 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import React from 'react';
-
-import {Layout} from 'antd';
-import './App.less';
-import NavBar from './components/NavBar/NavBar';
-import Breadcrumbs from './components/Breadcrumbs/Breadcrumbs';
-import {HashRouter as Router, Switch, Route, Redirect} from 'react-router-dom';
-import {routes} from './routes';
-import {MakeRouteWithSubRoutes} from './makeRouteWithSubRoutes';
-
-const classNames = require('classnames');
-const {
- Header, Content, Footer
-} = Layout;
-
-interface Props {
-}
-
-interface State {
- collapsed: boolean;
-}
-
-class App extends React.Component<Props, State> {
-
- constructor(props: Props) {
- super(props);
-
- this.state = {collapsed: false};
- }
-
- onCollapse = (collapsed: boolean) => {
- this.setState({collapsed});
- };
-
- render() {
- const {collapsed} = this.state;
- const layoutClass = classNames('content-layout', {'sidebar-collapsed': collapsed});
-
- return (
- <Router>
- <Layout style={{minHeight: '100vh'}}>
- <NavBar collapsed={collapsed} onCollapse={this.onCollapse}/>
- <Layout className={layoutClass}>
- <Header>
- <div style={{margin: '16px 0'}}>
- <Breadcrumbs/>
- </div>
- </Header>
- <Content style={{margin: '0 16px 0', overflow: 'initial'}}>
- <Switch>
- <Route exact path="/">
- <Redirect to="/Overview"/>
- </Route>
- {
- routes.map(
- (route, index) => <MakeRouteWithSubRoutes key={index} {...route} />
- )
- }
- </Switch>
- </Content>
- <Footer style={{textAlign: 'center'}}>
- </Footer>
- </Layout>
- </Layout>
- </Router>
- );
- }
-}
-
-export default App;
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/App.less b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/app.less
similarity index 90%
rename from hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/App.less
rename to hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/app.less
index 2476db4..43fc19d 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/App.less
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/app.less
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -17,7 +17,7 @@
*/
@import "~antd/es/style/themes/default.less";
-@import "./components/NavBar/NavBar.less";
+@import "components/navBar/navBar.less";
.ant-layout-header {
padding: 0 20px;
@@ -126,13 +126,13 @@
.hexagon-shape(20, 12, @orange-7);
}
-.icon-text(@content) {
+.icon-text(@content, @fontcolor) {
text-align: center;
font-size: 12px;
font-weight: 700;
position: relative;
top: -3px;
- color: #fff;
+ color: @fontcolor;
&:after {
content: @content;
position: absolute;
@@ -140,7 +140,7 @@
top: 4px;
width: 10px;
height: 0;
- color: white;
+ color: @fontcolor;
font-size: 12px;
z-index: 2;
}
@@ -148,14 +148,23 @@
.icon-text-three-dots {
// In Unicode, \2026 is the horizontal ellipsis (...)
- .icon-text("\2026");
+ .icon-text("\2026", #fff);
+}
+
+.icon-text-three-dots-leader {
+ // In Unicode, \2026 is the horizontal ellipsis (...)
+ .icon-text("\2026", #ffde36);
}
.icon-text-one-dot {
- .icon-text(".");
+ .icon-text(".", #ffde36);
}
.replication-icon {
display: inline-block;
margin-right: 5px;
}
+
+.pointer {
+ cursor: pointer;
+}
\ No newline at end of file
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/App.test.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/app.test.tsx
similarity index 94%
rename from hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/App.test.tsx
rename to hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/app.test.tsx
index 0205e74..ad78746 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/App.test.tsx
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/app.test.tsx
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -18,10 +18,10 @@
import React from 'react';
import ReactDOM from 'react-dom';
-import App from './App';
+import App from './app';
it('renders without crashing', () => {
const div = document.createElement('div');
- ReactDOM.render(<App />, div);
+ ReactDOM.render(<App/>, div);
ReactDOM.unmountComponentAtNode(div);
});
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/app.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/app.tsx
new file mode 100644
index 0000000..a21d928
--- /dev/null
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/app.tsx
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+
+import {Layout} from 'antd';
+import './app.less';
+import NavBar from './components/navBar/navBar';
+import Breadcrumbs from './components/breadcrumbs/breadcrumbs';
+import {HashRouter as Router, Switch, Route, Redirect} from 'react-router-dom';
+import {routes} from './routes';
+import {MakeRouteWithSubRoutes} from './makeRouteWithSubRoutes';
+import classNames from 'classnames';
+
+const {
+ Header, Content, Footer
+} = Layout;
+
+interface IAppState {
+ collapsed: boolean;
+}
+
+class App extends React.Component<Record<string, object>, IAppState> {
+ constructor(props = {}) {
+ super(props);
+ this.state = {collapsed: false};
+ }
+
+ onCollapse = (collapsed: boolean) => {
+ this.setState({collapsed});
+ };
+
+ render() {
+ const {collapsed} = this.state;
+ const layoutClass = classNames('content-layout', {'sidebar-collapsed': collapsed});
+
+ return (
+ <Router>
+ <Layout style={{minHeight: '100vh'}}>
+ <NavBar collapsed={collapsed} onCollapse={this.onCollapse}/>
+ <Layout className={layoutClass}>
+ <Header>
+ <div style={{margin: '16px 0'}}>
+ <Breadcrumbs/>
+ </div>
+ </Header>
+ <Content style={{margin: '0 16px 0', overflow: 'initial'}}>
+ <Switch>
+ <Route exact path='/'>
+ <Redirect to='/Overview'/>
+ </Route>
+ {
+ routes.map(
+ (route, index) => <MakeRouteWithSubRoutes key={index} {...route}/>
+ )
+ }
+ </Switch>
+ </Content>
+ <Footer style={{textAlign: 'center'}}/>
+ </Layout>
+ </Layout>
+ </Router>
+ );
+ }
+}
+
+export default App;
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/NavBar/NavBar.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/NavBar/NavBar.tsx
deleted file mode 100644
index 94fd8ae..0000000
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/NavBar/NavBar.tsx
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import React from 'react';
-import logo from '../../logo.png';
-import {Layout, Menu, Icon} from 'antd';
-import './NavBar.less';
-import {withRouter, Link} from 'react-router-dom';
-import {RouteComponentProps} from 'react-router';
-
-const {Sider} = Layout;
-
-interface NavBarProps extends RouteComponentProps<any> {
- collapsed: boolean;
- onCollapse: (arg: boolean) => void;
-}
-
-class NavBar extends React.Component<NavBarProps> {
- render() {
- const {location} = this.props;
- return (
- <Sider
- collapsible
- collapsed={this.props.collapsed}
- collapsedWidth={50}
- onCollapse={this.props.onCollapse}
- style={{
- overflow: 'auto', height: '100vh', position: 'fixed', left: 0,
- }}
- >
- <div className="logo">
- <img src={logo} alt="Ozone Recon Logo" width={32} height={32}/>
- <span className="logo-text">Ozone Recon</span>
- </div>
- <Menu theme="dark" defaultSelectedKeys={['/Dashboard']}
- mode="inline" selectedKeys={[location.pathname]}>
- <Menu.Item key="/Overview">
- <Icon type="dashboard"/>
- <span>Overview</span>
- <Link to="/Overview"/>
- </Menu.Item>
- <Menu.Item key="/Datanodes">
- <Icon type="cluster"/>
- <span>Datanodes</span>
- <Link to="/Datanodes"/>
- </Menu.Item>
- <Menu.Item key="/Pipelines">
- <Icon type="deployment-unit"/>
- <span>Pipelines</span>
- <Link to="/Pipelines"/>
- </Menu.Item>
- </Menu>
- </Sider>
- );
- }
-}
-
-export default withRouter(NavBar);
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/OverviewCard/OverviewCard.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/OverviewCard/OverviewCard.tsx
deleted file mode 100644
index 579f782..0000000
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/OverviewCard/OverviewCard.tsx
+++ /dev/null
@@ -1,100 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import React, {ReactElement} from 'react';
-import {Icon, Card, Row, Col} from 'antd';
-import {withRouter, Link} from 'react-router-dom';
-import {RouteComponentProps} from 'react-router';
-import StorageBar from "../StorageBar/StorageBar";
-import {StorageReport} from "types/datanode.types";
-import './OverviewCard.less';
-
-const {Meta} = Card;
-
-interface OverviewCardProps extends RouteComponentProps<any> {
- icon: string;
- data: string | ReactElement;
- title: string;
- hoverable?: boolean;
- loading?: boolean;
- linkToUrl?: string;
- storageReport?: StorageReport;
- error?: boolean;
-}
-
-const defaultProps = {
- hoverable: false,
- loading: false,
- linkToUrl: '',
- error: false
-};
-
-interface OverviewCardWrapperProps {
- linkToUrl: string;
-}
-
-class OverviewCardWrapper extends React.Component<OverviewCardWrapperProps> {
- render() {
- let {linkToUrl, children} = this.props;
- if (linkToUrl) {
- return <Link to={linkToUrl}>
- {children}
- </Link>
- } else {
- return children;
- }
- }
-}
-
-class OverviewCard extends React.Component<OverviewCardProps> {
- static defaultProps = defaultProps;
-
- render() {
- let {icon, data, title, loading, hoverable, storageReport, linkToUrl, error} = this.props;
- let meta = <Meta title={data} description={title}/>;
- const errorClass = error ? 'card-error' : '';
- if (storageReport) {
- meta = <div className="ant-card-percentage">
- {meta}
- <div className="storage-bar">
- <StorageBar total={storageReport.capacity} used={storageReport.used} remaining={storageReport.remaining} showMeta={false}/>
- </div>
- </div>;
- }
- linkToUrl = linkToUrl || '';
-
- return (
- <OverviewCardWrapper linkToUrl={linkToUrl}>
- <Card className={`overview-card ${errorClass}`} loading={loading} hoverable={hoverable}>
- <Row type="flex" justify="space-between">
- <Col span={18}>
- <Row>
- {meta}
- </Row>
- </Col>
- <Col span={6}>
- <Icon type={icon} style={{"fontSize": "50px", "float": "right"}}/>
- </Col>
- </Row>
- </Card>
- </OverviewCardWrapper>
- );
- }
-}
-
-export default withRouter(OverviewCard);
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/StorageBar/StorageBar.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/StorageBar/StorageBar.tsx
deleted file mode 100644
index 7eab27a..0000000
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/StorageBar/StorageBar.tsx
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import React from 'react';
-import {Icon, Progress} from 'antd';
-import {withRouter} from 'react-router-dom';
-import {RouteComponentProps} from 'react-router';
-import {FilledIcon} from "utils/themeIcons";
-import Tooltip from "antd/lib/tooltip";
-import {getCapacityPercent} from "utils/common";
-import filesize from "filesize";
-import './StorageBar.less';
-
-const size = filesize.partial({standard: 'iec', round: 1});
-
-interface StorageBarProps extends RouteComponentProps<any> {
- total: number;
- used: number;
- remaining: number;
- showMeta?: boolean;
-}
-
-const defaultProps = {
- total: 0,
- used: 0,
- remaining: 0,
- showMeta: true
-};
-
-class StorageBar extends React.Component<StorageBarProps> {
- static defaultProps = defaultProps;
-
- render() {
- let {total, used, remaining, showMeta} = this.props;
- const nonOzoneUsed = total - remaining - used;
- const totalUsed = total - remaining;
- const tooltip = <div>
- <div><Icon component={FilledIcon} className="ozone-used-bg"/> Ozone Used ({size(used)})</div>
- <div><Icon component={FilledIcon} className="non-ozone-used-bg"/> Non Ozone Used ({size(nonOzoneUsed)})</div>
- <div><Icon component={FilledIcon} className="remaining-bg"/> Remaining ({size(remaining)})</div>
- </div>;
- const metaElement = showMeta ? <div>{size(used)} + {size(nonOzoneUsed)} / {size(total)}</div> : null;
- return <div className={"storage-cell-container"}>
- <Tooltip title={tooltip} placement="bottomLeft">
- {metaElement}
- <Progress strokeLinecap="square"
- percent={getCapacityPercent(totalUsed, total)}
- successPercent={getCapacityPercent(used, total)}
- className="capacity-bar" strokeWidth={3}/>
- </Tooltip>
- </div>
- }
-}
-
-export default withRouter(StorageBar);
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Pipelines/Pipelines.less b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/autoReloadPanel/autoReloadPanel.less
similarity index 84%
copy from hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Pipelines/Pipelines.less
copy to hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/autoReloadPanel/autoReloadPanel.less
index 8edff71..bfc5326 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Pipelines/Pipelines.less
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/autoReloadPanel/autoReloadPanel.less
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -16,8 +16,13 @@
* limitations under the License.
*/
-.pipelines-container {
- .content-div {
-
+.auto-reload-panel {
+ float: right;
+ font-size: 14px;
+ font-weight: normal;
+ padding-top: 3px;
+ .toggle-switch {
+ margin-top: -2px;
}
-}
\ No newline at end of file
+}
+
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/autoReloadPanel/autoReloadPanel.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/autoReloadPanel/autoReloadPanel.tsx
new file mode 100644
index 0000000..fee9a88
--- /dev/null
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/autoReloadPanel/autoReloadPanel.tsx
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+
+import {Tooltip, Button, Switch} from 'antd';
+import './autoReloadPanel.less';
+import {withRouter} from 'react-router-dom';
+import {RouteComponentProps} from 'react-router';
+import moment from 'moment';
+
+interface IAutoReloadPanelProps extends RouteComponentProps<object> {
+ onReload: () => void;
+ lastUpdated: number;
+ isLoading: boolean;
+ togglePolling: (isEnabled: boolean) => void;
+}
+
+class AutoReloadPanel extends React.Component<IAutoReloadPanelProps> {
+ autoReloadToggleHandler = (checked: boolean, _event: Event) => {
+ const {togglePolling} = this.props;
+ togglePolling(checked);
+ };
+
+ render() {
+ const {onReload, lastUpdated, isLoading} = this.props;
+ const lastUpdatedText = lastUpdated === 0 ? 'NA' :
+ (
+ <Tooltip
+ placement='bottom' title={moment(lastUpdated).format('lll')}
+ >
+ {moment(lastUpdated).format('LT')}
+ </Tooltip>
+ );
+ return (
+ <div className='auto-reload-panel'>
+ Auto Reload
+ <Switch defaultChecked size='small' className='toggle-switch' onChange={this.autoReloadToggleHandler}/>
+ | Last updated at {lastUpdatedText}
+ <Button shape='circle' icon='reload' size='small' loading={isLoading} onClick={onReload}/>
+ </div>
+ );
+ }
+}
+
+export default withRouter(AutoReloadPanel);
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/Breadcrumbs/Breadcrumbs.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/breadcrumbs/breadcrumbs.tsx
similarity index 74%
rename from hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/Breadcrumbs/Breadcrumbs.tsx
rename to hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/breadcrumbs/breadcrumbs.tsx
index 383d111..cc976d5 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/Breadcrumbs/Breadcrumbs.tsx
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/breadcrumbs/breadcrumbs.tsx
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -20,37 +20,31 @@
import {Breadcrumb, Icon} from 'antd';
import {withRouter, Link} from 'react-router-dom';
import {RouteComponentProps} from 'react-router';
-import {breadcrumbNameMap} from '../../constants/breadcrumbs.constants';
-
-interface Props extends RouteComponentProps<any> {
- collapsed: boolean;
- onCollapse: (arg: boolean) => void;
-}
+import {breadcrumbNameMap} from 'constants/breadcrumbs.constants';
class Breadcrumbs extends React.Component<RouteComponentProps> {
-
render() {
const {location} = this.props;
const pathSnippets = location.pathname.split('/').filter(i => i);
const extraBreadcrumbItems = pathSnippets.map((_, index) => {
const url = `/${pathSnippets.slice(0, index + 1).join('/')}`;
return (
- <Breadcrumb.Item key={url}>
- <Link to={url}>
- {breadcrumbNameMap[url]}
- </Link>
- </Breadcrumb.Item>
+ <Breadcrumb.Item key={url}>
+ <Link to={url}>
+ {breadcrumbNameMap[url]}
+ </Link>
+ </Breadcrumb.Item>
);
});
const breadcrumbItems = [(
- <Breadcrumb.Item key="home">
- <Link to="/"><Icon type="home"/></Link>
- </Breadcrumb.Item>
+ <Breadcrumb.Item key='home'>
+ <Link to='/'><Icon type='home'/></Link>
+ </Breadcrumb.Item>
)].concat(extraBreadcrumbItems);
return (
- <Breadcrumb>
- {breadcrumbItems}
- </Breadcrumb>
+ <Breadcrumb>
+ {breadcrumbItems}
+ </Breadcrumb>
);
}
}
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/NavBar/NavBar.less b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/navBar/navBar.less
similarity index 99%
rename from hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/NavBar/NavBar.less
rename to hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/navBar/navBar.less
index 6a4da21..b1570b6 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/NavBar/NavBar.less
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/navBar/navBar.less
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/navBar/navBar.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/navBar/navBar.tsx
new file mode 100644
index 0000000..b60a73f
--- /dev/null
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/navBar/navBar.tsx
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import logo from '../../logo.png';
+import {Layout, Menu, Icon} from 'antd';
+import './navBar.less';
+import {withRouter, Link} from 'react-router-dom';
+import {RouteComponentProps} from 'react-router';
+
+const {Sider} = Layout;
+
+interface INavBarProps extends RouteComponentProps<object> {
+ collapsed: boolean;
+ onCollapse: (arg: boolean) => void;
+}
+
+class NavBar extends React.Component<INavBarProps> {
+ render() {
+ const {location} = this.props;
+ return (
+ <Sider
+ collapsible
+ collapsed={this.props.collapsed}
+ collapsedWidth={50}
+ style={{
+ overflow: 'auto', height: '100vh', position: 'fixed', left: 0
+ }}
+ onCollapse={this.props.onCollapse}
+ >
+ <div className='logo'>
+ <img src={logo} alt='Ozone Recon Logo' width={32} height={32}/>
+ <span className='logo-text'>Ozone Recon</span>
+ </div>
+ <Menu
+ theme='dark' defaultSelectedKeys={['/Dashboard']}
+ mode='inline' selectedKeys={[location.pathname]}
+ >
+ <Menu.Item key='/Overview'>
+ <Icon type='dashboard'/>
+ <span>Overview</span>
+ <Link to='/Overview'/>
+ </Menu.Item>
+ <Menu.Item key='/Datanodes'>
+ <Icon type='cluster'/>
+ <span>Datanodes</span>
+ <Link to='/Datanodes'/>
+ </Menu.Item>
+ <Menu.Item key='/Pipelines'>
+ <Icon type='deployment-unit'/>
+ <span>Pipelines</span>
+ <Link to='/Pipelines'/>
+ </Menu.Item>
+ </Menu>
+ </Sider>
+ );
+ }
+}
+
+export default withRouter(NavBar);
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/OverviewCard/OverviewCard.less b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/overviewCard/overviewCard.less
similarity index 99%
rename from hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/OverviewCard/OverviewCard.less
rename to hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/overviewCard/overviewCard.less
index 89bab0c..58850fb 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/OverviewCard/OverviewCard.less
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/overviewCard/overviewCard.less
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/overviewCard/overviewCard.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/overviewCard/overviewCard.tsx
new file mode 100644
index 0000000..df15f8a
--- /dev/null
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/overviewCard/overviewCard.tsx
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React, {ReactElement} from 'react';
+import {Icon, Card, Row, Col} from 'antd';
+import {withRouter, Link} from 'react-router-dom';
+import {RouteComponentProps} from 'react-router';
+import StorageBar from '../storageBar/storageBar';
+import {IStorageReport} from 'types/datanode.types';
+import './overviewCard.less';
+
+const {Meta} = Card;
+
+interface IOverviewCardProps extends RouteComponentProps<object> {
+ icon: string;
+ data: string | ReactElement;
+ title: string;
+ hoverable?: boolean;
+ loading?: boolean;
+ linkToUrl?: string;
+ storageReport?: IStorageReport;
+ error?: boolean;
+}
+
+const defaultProps = {
+ hoverable: false,
+ loading: false,
+ linkToUrl: '',
+ error: false
+};
+
+interface IOverviewCardWrapperProps {
+ linkToUrl: string;
+}
+
+class OverviewCardWrapper extends React.Component<IOverviewCardWrapperProps> {
+ render() {
+ const {linkToUrl, children} = this.props;
+ if (linkToUrl) {
+ return (
+ <Link to={linkToUrl}>
+ {children}
+ </Link>
+ );
+ }
+
+ return children;
+ }
+}
+
+class OverviewCard extends React.Component<IOverviewCardProps> {
+ static defaultProps = defaultProps;
+
+ render() {
+ let {icon, data, title, loading, hoverable, storageReport, linkToUrl, error} = this.props;
+ let meta = <Meta title={data} description={title}/>;
+ const errorClass = error ? 'card-error' : '';
+ if (storageReport) {
+ meta = (
+ <div className='ant-card-percentage'>
+ {meta}
+ <div className='storage-bar'>
+ <StorageBar total={storageReport.capacity} used={storageReport.used} remaining={storageReport.remaining} showMeta={false}/>
+ </div>
+ </div>
+ );
+ }
+
+ linkToUrl = linkToUrl ? linkToUrl : '';
+
+ return (
+ <OverviewCardWrapper linkToUrl={linkToUrl}>
+ <Card className={`overview-card ${errorClass}`} loading={loading} hoverable={hoverable}>
+ <Row type='flex' justify='space-between'>
+ <Col span={18}>
+ <Row>
+ {meta}
+ </Row>
+ </Col>
+ <Col span={6}>
+ <Icon type={icon} style={{fontSize: '50px', float: 'right'}}/>
+ </Col>
+ </Row>
+ </Card>
+ </OverviewCardWrapper>
+ );
+ }
+}
+
+export default withRouter(OverviewCard);
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/StorageBar/StorageBar.less b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/storageBar/storageBar.less
similarity index 99%
rename from hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/StorageBar/StorageBar.less
rename to hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/storageBar/storageBar.less
index e93d1b4..b2dddbc 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/StorageBar/StorageBar.less
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/storageBar/storageBar.less
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/storageBar/storageBar.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/storageBar/storageBar.tsx
new file mode 100644
index 0000000..10decce
--- /dev/null
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/storageBar/storageBar.tsx
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import {Icon, Progress} from 'antd';
+import {withRouter} from 'react-router-dom';
+import {RouteComponentProps} from 'react-router';
+import {FilledIcon} from 'utils/themeIcons';
+import Tooltip from 'antd/lib/tooltip';
+import {getCapacityPercent} from 'utils/common';
+import filesize from 'filesize';
+import './storageBar.less';
+
+const size = filesize.partial({standard: 'iec', round: 1});
+
+interface IStorageBarProps extends RouteComponentProps<object> {
+ total: number;
+ used: number;
+ remaining: number;
+ showMeta?: boolean;
+}
+
+const defaultProps = {
+ total: 0,
+ used: 0,
+ remaining: 0,
+ showMeta: true
+};
+
+class StorageBar extends React.Component<IStorageBarProps> {
+ static defaultProps = defaultProps;
+
+ render() {
+ const {total, used, remaining, showMeta} = this.props;
+ const nonOzoneUsed = total - remaining - used;
+ const totalUsed = total - remaining;
+ const tooltip = (
+ <div>
+ <div><Icon component={FilledIcon} className='ozone-used-bg'/> Ozone Used ({size(used)})</div>
+ <div><Icon component={FilledIcon} className='non-ozone-used-bg'/> Non Ozone Used ({size(nonOzoneUsed)})</div>
+ <div><Icon component={FilledIcon} className='remaining-bg'/> Remaining ({size(remaining)})</div>
+ </div>
+ );
+ const metaElement = showMeta ? <div>{size(used)} + {size(nonOzoneUsed)} / {size(total)}</div> : null;
+ return (
+ <div className='storage-cell-container'>
+ <Tooltip title={tooltip} placement='bottomLeft'>
+ {metaElement}
+ <Progress
+ strokeLinecap='square'
+ percent={getCapacityPercent(totalUsed, total)}
+ successPercent={getCapacityPercent(used, total)}
+ className='capacity-bar' strokeWidth={3}/>
+ </Tooltip>
+ </div>
+ );
+ }
+}
+
+export default withRouter(StorageBar);
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/MissingContainers/MissingContainers.less b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/constants/autoReload.constants.tsx
similarity index 87%
copy from hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/MissingContainers/MissingContainers.less
copy to hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/constants/autoReload.constants.tsx
index 5d5f1e3..c78b6d8 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/MissingContainers/MissingContainers.less
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/constants/autoReload.constants.tsx
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -15,3 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
+// default global auto reload interval: 60 seconds
+export const AUTO_RELOAD_INTERVAL_DEFAULT = 60 * 1000;
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/constants/breadcrumbs.constants.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/constants/breadcrumbs.constants.tsx
index 93773b9..6e7a7d0 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/constants/breadcrumbs.constants.tsx
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/constants/breadcrumbs.constants.tsx
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/index.less b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/index.less
index 1b94f4e..78912c9 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/index.less
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/index.less
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/index.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/index.tsx
index a3e450c..0e2f2a4 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/index.tsx
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/index.tsx
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -19,12 +19,6 @@
import React from 'react';
import ReactDOM from 'react-dom';
import './index.less';
-import App from './App';
-import * as serviceWorker from './serviceWorker';
+import App from './app';
-ReactDOM.render(<App />, document.getElementById('root'));
-
-// If you want your app to work offline and load faster, you can change
-// unregister() to register() below. Note this comes with some pitfalls.
-// Learn more about service workers: https://bit.ly/CRA-PWA
-serviceWorker.unregister();
+ReactDOM.render(<App/>, document.querySelector('#root'));
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/makeRouteWithSubRoutes.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/makeRouteWithSubRoutes.tsx
index aa34876..237c372 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/makeRouteWithSubRoutes.tsx
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/makeRouteWithSubRoutes.tsx
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -17,16 +17,16 @@
*/
import React from 'react';
-import { Route } from 'react-router-dom';
-import { IRoute } from './types/routes.types';
+import {Route} from 'react-router-dom';
+import {IRoute} from './types/routes.types';
export const MakeRouteWithSubRoutes = (route: IRoute) => {
return (
- <Route
- path={route.path}
- render={props => (
- <route.component {...props} routes={route.routes} />
- )}
- />
+ <Route
+ path={route.path}
+ render={props => (
+ <route.component {...props} routes={route.routes}/>
+ )}
+ />
);
};
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/react-app-env.d.ts b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/react-app-env.d.ts
index 15f01c4..95dfdcf 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/react-app-env.d.ts
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/react-app-env.d.ts
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/routes.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/routes.tsx
index cbafba4..999efd8 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/routes.tsx
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/routes.tsx
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -16,32 +16,32 @@
* limitations under the License.
*/
-import {Overview} from './views/Overview/Overview';
-import {Datanodes} from './views/Datanodes/Datanodes';
-import {Pipelines} from "./views/Pipelines/Pipelines";
-import {NotFound} from './views/NotFound/NotFound';
-import {IRoute} from "./types/routes.types";
-import {MissingContainers} from "./views/MissingContainers/MissingContainers";
+import {Overview} from './views/overview/overview';
+import {Datanodes} from './views/datanodes/datanodes';
+import {Pipelines} from './views/pipelines/pipelines';
+import {NotFound} from './views/notFound/notFound';
+import {IRoute} from './types/routes.types';
+import {MissingContainers} from './views/missingContainers/missingContainers';
export const routes: IRoute[] = [
{
- path: "/Overview",
+ path: '/Overview',
component: Overview
},
{
- path: "/Datanodes",
+ path: '/Datanodes',
component: Datanodes
},
{
- path: "/Pipelines",
+ path: '/Pipelines',
component: Pipelines
},
{
- path: "/MissingContainers",
+ path: '/MissingContainers',
component: MissingContainers
},
{
- path: "/:NotFound",
- component: NotFound,
+ path: '/:NotFound',
+ component: NotFound
}
];
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/serviceWorker.ts b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/serviceWorker.ts
deleted file mode 100644
index 47bb33b..0000000
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/serviceWorker.ts
+++ /dev/null
@@ -1,161 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This optional code is used to register a service worker.
-// register() is not called by default.
-
-// This lets the app load faster on subsequent visits in production, and gives
-// it offline capabilities. However, it also means that developers (and users)
-// will only see deployed updates on subsequent visits to a page, after all the
-// existing tabs open on the page have been closed, since previously cached
-// resources are updated in the background.
-
-// To learn more about the benefits of this model and instructions on how to
-// opt-in, read https://bit.ly/CRA-PWA
-
-const isLocalhost = Boolean(
- window.location.hostname === 'localhost' ||
- // [::1] is the IPv6 localhost address.
- window.location.hostname === '[::1]' ||
- // 127.0.0.1/8 is considered localhost for IPv4.
- window.location.hostname.match(
- /^127(?:\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}$/
- )
-);
-
-type Config = {
- onSuccess?: (registration: ServiceWorkerRegistration) => void;
- onUpdate?: (registration: ServiceWorkerRegistration) => void;
-};
-
-export function register(config?: Config) {
- if (process.env.NODE_ENV === 'production' && 'serviceWorker' in navigator) {
- // The URL constructor is available in all browsers that support SW.
- const publicUrl = new URL(
- (process as { env: { [key: string]: string } }).env.PUBLIC_URL,
- window.location.href
- );
- if (publicUrl.origin !== window.location.origin) {
- // Our service worker won't work if PUBLIC_URL is on a different origin
- // from what our page is served on. This might happen if a CDN is used to
- // serve assets; see https://github.com/facebook/create-react-app/issues/2374
- return;
- }
-
- window.addEventListener('load', () => {
- const swUrl = `${process.env.PUBLIC_URL}/service-worker.js`;
-
- if (isLocalhost) {
- // This is running on localhost. Let's check if a service worker still exists or not.
- checkValidServiceWorker(swUrl, config);
-
- // Add some additional logging to localhost, pointing developers to the
- // service worker/PWA documentation.
- navigator.serviceWorker.ready.then(() => {
- console.log(
- 'This web app is being served cache-first by a service ' +
- 'worker. To learn more, visit https://bit.ly/CRA-PWA'
- );
- });
- } else {
- // Is not localhost. Just register service worker
- registerValidSW(swUrl, config);
- }
- });
- }
-}
-
-function registerValidSW(swUrl: string, config?: Config) {
- navigator.serviceWorker
- .register(swUrl)
- .then(registration => {
- registration.onupdatefound = () => {
- const installingWorker = registration.installing;
- if (installingWorker == null) {
- return;
- }
- installingWorker.onstatechange = () => {
- if (installingWorker.state === 'installed') {
- if (navigator.serviceWorker.controller) {
- // At this point, the updated precached content has been fetched,
- // but the previous service worker will still serve the older
- // content until all client tabs are closed.
- console.log(
- 'New content is available and will be used when all ' +
- 'tabs for this page are closed. See https://bit.ly/CRA-PWA.'
- );
-
- // Execute callback
- if (config && config.onUpdate) {
- config.onUpdate(registration);
- }
- } else {
- // At this point, everything has been precached.
- // It's the perfect time to display a
- // "Content is cached for offline use." message.
- console.log('Content is cached for offline use.');
-
- // Execute callback
- if (config && config.onSuccess) {
- config.onSuccess(registration);
- }
- }
- }
- };
- };
- })
- .catch(error => {
- console.error('Error during service worker registration:', error);
- });
-}
-
-function checkValidServiceWorker(swUrl: string, config?: Config) {
- // Check if the service worker can be found. If it can't reload the page.
- fetch(swUrl)
- .then(response => {
- // Ensure service worker exists, and that we really are getting a JS file.
- const contentType = response.headers.get('content-type');
- if (
- response.status === 404 ||
- (contentType != null && contentType.indexOf('javascript') === -1)
- ) {
- // No service worker found. Probably a different app. Reload the page.
- navigator.serviceWorker.ready.then(registration => {
- registration.unregister().then(() => {
- window.location.reload();
- });
- });
- } else {
- // Service worker found. Proceed as normal.
- registerValidSW(swUrl, config);
- }
- })
- .catch(() => {
- console.log(
- 'No internet connection found. App is running in offline mode.'
- );
- });
-}
-
-export function unregister() {
- if ('serviceWorker' in navigator) {
- navigator.serviceWorker.ready.then(registration => {
- registration.unregister();
- });
- }
-}
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Pipelines/Pipelines.less b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/types/axios.types.tsx
similarity index 93%
copy from hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Pipelines/Pipelines.less
copy to hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/types/axios.types.tsx
index 8edff71..7cb69fa 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Pipelines/Pipelines.less
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/types/axios.types.tsx
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -16,8 +16,6 @@
* limitations under the License.
*/
-.pipelines-container {
- .content-div {
-
- }
-}
\ No newline at end of file
+export interface IAxiosResponse<T> {
+ data: T;
+}
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/types/datanode.types.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/types/datanode.types.tsx
index 5f8cfcd..ba8336b 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/types/datanode.types.tsx
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/types/datanode.types.tsx
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -16,9 +16,9 @@
* limitations under the License.
*/
-export type DatanodeStatus = "HEALTHY" | "STALE" | "DEAD" | "DECOMMISSIONING" | "DECOMMISSIONED";
+export type DatanodeStatus = 'HEALTHY' | 'STALE' | 'DEAD' | 'DECOMMISSIONING' | 'DECOMMISSIONED';
-export interface StorageReport {
+export interface IStorageReport {
capacity: number;
used: number;
remaining: number;
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/types/routes.types.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/types/routes.types.tsx
index 7e12d80..d055141 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/types/routes.types.tsx
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/types/routes.types.tsx
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -16,8 +16,10 @@
* limitations under the License.
*/
+import React from 'react';
+
export interface IRoute {
path: string;
- component: any;
+ component: React.ElementType;
routes?: IRoute[];
}
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/utils/autoReloadHelper.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/utils/autoReloadHelper.tsx
new file mode 100644
index 0000000..20c8018
--- /dev/null
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/utils/autoReloadHelper.tsx
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {AUTO_RELOAD_INTERVAL_DEFAULT} from '../constants/autoReload.constants';
+
+class AutoReloadHelper {
+ loadData: () => void;
+ interval = 0;
+
+ constructor(loadData: () => void) {
+ this.loadData = loadData;
+ }
+
+ initPolling = () => {
+ this.loadData();
+ this.interval = window.setTimeout(this.initPolling, AUTO_RELOAD_INTERVAL_DEFAULT);
+ };
+
+ startPolling = () => {
+ this.stopPolling();
+ this.interval = window.setTimeout(this.initPolling, AUTO_RELOAD_INTERVAL_DEFAULT);
+ };
+
+ stopPolling = () => {
+ if (this.interval > 0) {
+ clearTimeout(this.interval);
+ }
+ };
+
+ handleAutoReloadToggle = (checked: boolean) => {
+ if (checked) {
+ this.startPolling();
+ } else {
+ this.stopPolling();
+ }
+ };
+}
+
+export {AutoReloadHelper};
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/utils/common.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/utils/common.tsx
index 97b68d8..d5aa0a0 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/utils/common.tsx
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/utils/common.tsx
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -16,4 +16,24 @@
* limitations under the License.
*/
-export const getCapacityPercent = (used: number, total: number) => Math.round((used / total) * 100);
\ No newline at end of file
+import moment from 'moment';
+import {notification} from 'antd';
+
+export const getCapacityPercent = (used: number, total: number) => Math.round((used / total) * 100);
+
+export const timeFormat = (time: number) => time > 0 ?
+ moment(time).format('lll') : 'NA';
+
+const showErrorNotification = (title: string, description: string) => {
+ const args = {
+ message: title,
+ description,
+ duration: 15
+ };
+ notification.error(args);
+};
+
+export const showDataFetchError = (error: string) => {
+ const title = 'Error while fetching data';
+ showErrorNotification(title, error);
+};
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/utils/themeIcons.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/utils/themeIcons.tsx
index 165e8d8..9868638 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/utils/themeIcons.tsx
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/utils/themeIcons.tsx
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -17,7 +17,7 @@
*/
import * as React from 'react';
-import {Tooltip} from "antd";
+import {Tooltip} from 'antd';
export class FilledIcon extends React.Component {
render() {
@@ -26,64 +26,75 @@
'704c0 53 43 96 96 96h704c53 0 96-43 96-96V16' +
'0c0-53-43-96-96-96z';
return (
- <svg {...(this.props as any)} viewBox="0 0 1024 1024">
- <path d={path} />
- </svg>
+ <svg {...(this.props as Record<string, object>)} viewBox='0 0 1024 1024'>
+ <path d={path}/>
+ </svg>
);
}
}
-interface RatisIconProps {
+interface IRatisIconProps {
replicationFactor: number;
+ isLeader: boolean;
}
-interface ReplicationIconProps {
+interface IReplicationIconProps {
replicationFactor: number;
replicationType: string;
+ leaderNode: string;
+ isLeader: boolean;
}
-export class RatisIcon extends React.PureComponent<RatisIconProps> {
+export class RatisIcon extends React.PureComponent<IRatisIconProps> {
render() {
- const {replicationFactor} = this.props;
- const textClass = replicationFactor >= 3 ? 'icon-text-three-dots' : 'icon-text-one-dot';
+ const {replicationFactor, isLeader} = this.props;
+ const threeFactorClass = isLeader ? 'icon-text-three-dots-leader' : 'icon-text-three-dots';
+ const textClass = replicationFactor >= 3 ? threeFactorClass : 'icon-text-one-dot';
return (
- <div className="ratis-icon">
- <div className={textClass}>R</div>
- </div>
- )
+ <div className='ratis-icon'>
+ <div className={textClass}>R</div>
+ </div>
+ );
}
}
export class StandaloneIcon extends React.PureComponent {
render() {
return (
- <div className="standalone-icon">
- <div className="icon-text-one-dot">S</div>
- </div>
- )
+ <div className='standalone-icon'>
+ <div className='icon-text-one-dot'>S</div>
+ </div>
+ );
}
}
-export class ReplicationIcon extends React.PureComponent<ReplicationIconProps> {
+export class ReplicationIcon extends React.PureComponent<IReplicationIconProps> {
render() {
- const {replicationType, replicationFactor} = this.props;
+ const {replicationType, replicationFactor, isLeader, leaderNode} = this.props;
// Assign icons only for RATIS and STAND_ALONE types
let icon = null;
if (replicationType === 'RATIS') {
- icon = <RatisIcon replicationFactor={replicationFactor}/>
+ icon = <RatisIcon replicationFactor={replicationFactor} isLeader={isLeader}/>;
} else if (replicationType === 'STAND_ALONE') {
- icon = <StandaloneIcon/>
+ icon = <StandaloneIcon/>;
}
+
// Wrap the icon in a tooltip
if (icon) {
- const tooltip = <div>
- <div>Replication Type: {replicationType}</div>
- <div>Replication Factor: {replicationFactor}</div>
- </div>;
- icon = <Tooltip title={tooltip} placement="right">
- <div className="replication-icon">{icon}</div>
- </Tooltip>;
+ const tooltip = (
+ <div>
+ <div>Replication Type: {replicationType}</div>
+ <div>Replication Factor: {replicationFactor}</div>
+ <div>Leader Node: {leaderNode}</div>
+ </div>
+ );
+ icon = (
+ <Tooltip title={tooltip} placement='right' className='pointer'>
+ <div className='replication-icon'>{icon}</div>
+ </Tooltip>
+ );
}
+
return icon;
}
}
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Datanodes/Datanodes.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Datanodes/Datanodes.tsx
deleted file mode 100644
index cbcdc88..0000000
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Datanodes/Datanodes.tsx
+++ /dev/null
@@ -1,198 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import React from 'react';
-import axios from 'axios';
-import {Table, Icon} from 'antd';
-import {PaginationConfig} from 'antd/lib/pagination';
-import moment from 'moment';
-import {ReplicationIcon} from 'utils/themeIcons';
-import StorageBar from "components/StorageBar/StorageBar";
-import {DatanodeStatus, StorageReport} from "types/datanode.types";
-import './Datanodes.less';
-
-interface DatanodeResponse {
- hostname: string;
- state: DatanodeStatus;
- lastHeartbeat: number;
- storageReport: StorageReport;
- pipelines: Pipeline[];
- containers: number;
-}
-
-interface DatanodesResponse {
- totalCount: number;
- datanodes: DatanodeResponse[];
-}
-
-interface Datanode {
- hostname: string;
- state: DatanodeStatus;
- lastHeartbeat: number;
- storageUsed: number;
- storageTotal: number;
- storageRemaining: number;
- pipelines: Pipeline[];
- containers: number;
-}
-
-interface Pipeline {
- pipelineID: string;
- replicationType: string;
- replicationFactor: number;
-}
-
-interface DatanodesState {
- loading: boolean;
- dataSource: Datanode[];
- totalCount: number;
-}
-
-const renderDatanodeStatus = (status: DatanodeStatus) => {
- const statusIconMap = {
- HEALTHY: <Icon type="check-circle" theme="filled" twoToneColor="#1da57a" className="icon-success"/>,
- STALE: <Icon type="hourglass" theme="filled" className="icon-warning"/>,
- DEAD: <Icon type="close-circle" theme="filled" className="icon-failure"/>,
- DECOMMISSIONING: <Icon type="warning" theme="filled" className="icon-warning"/>,
- DECOMMISSIONED: <Icon type="exclamation-circle" theme="filled" className="icon-failure"/>
- };
- const icon = status in statusIconMap ? statusIconMap[status] : '';
- return <span>{icon} {status}</span>;
-};
-
-const COLUMNS = [
- {
- title: 'Status',
- dataIndex: 'state',
- key: 'state',
- render: (text: DatanodeStatus) => renderDatanodeStatus(text),
- sorter: (a: Datanode, b: Datanode) => a.state.localeCompare(b.state)
- },
- {
- title: 'Hostname',
- dataIndex: 'hostname',
- key: 'hostname',
- sorter: (a: Datanode, b: Datanode) => a.hostname.localeCompare(b.hostname),
- defaultSortOrder: 'ascend' as const
- },
- {
- title: 'Storage Capacity',
- dataIndex: 'storageUsed',
- key: 'storageUsed',
- sorter: (a: Datanode, b: Datanode) => a.storageRemaining - b.storageRemaining,
- render: (text: string, record: Datanode) =>
- <StorageBar total={record.storageTotal} used={record.storageUsed}
- remaining={record.storageRemaining}/>
- },
- {
- title: 'Last Heartbeat',
- dataIndex: 'lastHeartbeat',
- key: 'lastHeartbeat',
- sorter: (a: Datanode, b: Datanode) => a.lastHeartbeat - b.lastHeartbeat,
- render: (heartbeat: number) => {
- return heartbeat > 0 ? moment(heartbeat).format('lll') : 'NA';
- }
- },
- {
- title: 'Pipeline ID(s)',
- dataIndex: 'pipelines',
- key: 'pipelines',
- render: (pipelines: Pipeline[]) => {
- return (<div>
- {
- pipelines.map((pipeline, index) =>
- <div key={index} className="pipeline-container">
- <ReplicationIcon replicationFactor={pipeline.replicationFactor} replicationType={pipeline.replicationType}/>
- {pipeline.pipelineID}
- </div>)
- }
- </div>);
- }
- },
- {
- title: 'Containers',
- dataIndex: 'containers',
- key: 'containers',
- sorter: (a: Datanode, b: Datanode) => a.containers - b.containers
- }
-];
-
-export class Datanodes extends React.Component<any, DatanodesState> {
-
- constructor(props: any) {
- super(props);
- this.state = {
- loading: false,
- dataSource: [],
- totalCount: 0
- }
- }
-
- componentDidMount(): void {
- // Fetch datanodes on component mount
- this.setState({
- loading: true
- });
- axios.get('/api/v1/datanodes').then(response => {
- const datanodesResponse: DatanodesResponse = response.data;
- const totalCount = datanodesResponse.totalCount;
- const datanodes: DatanodeResponse[] = datanodesResponse.datanodes;
- const dataSource: Datanode[] = datanodes.map(datanode => {
- return {
- hostname: datanode.hostname,
- state: datanode.state,
- lastHeartbeat: datanode.lastHeartbeat,
- storageUsed: datanode.storageReport.used,
- storageTotal: datanode.storageReport.capacity,
- storageRemaining: datanode.storageReport.remaining,
- pipelines: datanode.pipelines,
- containers: datanode.containers
- }
- });
- this.setState({
- loading: false,
- dataSource,
- totalCount
- });
- });
- }
-
- onShowSizeChange = (current: number, pageSize: number) => {
- // TODO: Implement this method once server side pagination is enabled
- console.log(current, pageSize);
- };
-
- render () {
- const {dataSource, loading, totalCount} = this.state;
- const paginationConfig: PaginationConfig = {
- showTotal: (total: number, range) => `${range[0]}-${range[1]} of ${total} datanodes`,
- showSizeChanger: true,
- onShowSizeChange: this.onShowSizeChange
- };
- return (
- <div className="datanodes-container">
- <div className="page-header">
- Datanodes ({totalCount})
- </div>
- <div className="content-div">
- <Table dataSource={dataSource} columns={COLUMNS} loading={loading} pagination={paginationConfig} rowKey="hostname"/>
- </div>
- </div>
- );
- }
-}
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/MissingContainers/MissingContainers.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/MissingContainers/MissingContainers.tsx
deleted file mode 100644
index f97ec5e..0000000
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/MissingContainers/MissingContainers.tsx
+++ /dev/null
@@ -1,218 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import React from 'react';
-import axios from 'axios';
-import {Table} from 'antd';
-import './MissingContainers.less';
-import {PaginationConfig} from "antd/lib/pagination";
-import prettyBytes from "pretty-bytes";
-import moment from "moment";
-
-interface MissingContainerResponse {
- id: number;
- keys: number;
- datanodes: string[];
-}
-
-interface MissingContainersResponse {
- totalCount: number;
- containers: MissingContainerResponse[];
-}
-
-interface KeyResponse {
- Volume: string;
- Bucket: string;
- Key: string;
- DataSize: number;
- Versions: number[];
- Blocks: any;
- CreationTime: string;
- ModificationTime: string;
-}
-
-interface ContainerKeysResponse {
- totalCount: number;
- keys: KeyResponse[];
-}
-
-const COLUMNS = [
- {
- title: 'Container ID',
- dataIndex: 'id',
- key: 'id'
- },
- {
- title: 'No. of Keys',
- dataIndex: 'keys',
- key: 'keys'
- },
- {
- title: 'Datanodes',
- dataIndex: 'datanodes',
- key: 'datanodes',
- render: (datanodes: string[]) => <div>{datanodes.map(datanode => <div key={datanode}>{datanode}</div>)}</div>
- }
-];
-
-const KEY_TABLE_COLUMNS = [
- {
- title: 'Volume',
- dataIndex: 'Volume',
- key: 'Volume'
- },
- {
- title: 'Bucket',
- dataIndex: 'Bucket',
- key: 'Bucket'
- },
- {
- title: 'Key',
- dataIndex: 'Key',
- key: 'Key'
- },
- {
- title: 'Size',
- dataIndex: 'DataSize',
- key: 'DataSize',
- render: (dataSize: number) => <div>{prettyBytes(dataSize)}</div>
- },
- {
- title: 'Date Created',
- dataIndex: 'CreationTime',
- key: 'CreationTime',
- render: (date: number) => moment(date).format('lll')
- },
- {
- title: 'Date Modified',
- dataIndex: 'ModificationTime',
- key: 'ModificationTime',
- render: (date: number) => moment(date).format('lll')
- }
-];
-
-interface ExpandedRow {
- [key: number]: ExpandedRowState
-}
-
-interface ExpandedRowState {
- containerId: number;
- loading: boolean;
- dataSource: KeyResponse[];
- totalCount: number;
-}
-
-interface MissingContainersState {
- loading: boolean;
- dataSource: MissingContainerResponse[];
- totalCount: number;
- expandedRowData: ExpandedRow
-}
-
-export class MissingContainers extends React.Component<any, MissingContainersState> {
-
- constructor(props: any) {
- super(props);
- this.state = {
- loading: false,
- dataSource: [],
- totalCount: 0,
- expandedRowData: {}
- }
- }
-
- componentDidMount(): void {
- // Fetch missing containers on component mount
- this.setState({
- loading: true
- });
- axios.get('/api/v1/missingContainers').then(response => {
- const missingContainersResponse: MissingContainersResponse = response.data;
- const totalCount = missingContainersResponse.totalCount;
- const missingContainers: MissingContainerResponse[] = missingContainersResponse.containers;
- this.setState({
- loading: false,
- dataSource: missingContainers,
- totalCount: totalCount
- });
- });
- }
-
- onShowSizeChange = (current: number, pageSize: number) => {
- // TODO: Implement this method once server side pagination is enabled
- console.log(current, pageSize);
- };
-
- onRowExpandClick = (expanded: boolean, record: MissingContainerResponse) => {
- if (expanded) {
- this.setState(({expandedRowData}) => {
- const expandedRowState: ExpandedRowState = expandedRowData[record.id] ?
- Object.assign({}, expandedRowData[record.id], {loading: true}) :
- {containerId: record.id, loading: true, dataSource: [], totalCount: 0};
- return {
- expandedRowData: Object.assign({}, expandedRowData, {[record.id]: expandedRowState})
- }
- });
- axios.get(`/api/v1/containers/${record.id}/keys`).then(response => {
- const containerKeysResponse: ContainerKeysResponse = response.data;
- this.setState(({expandedRowData}) => {
- const expandedRowState: ExpandedRowState =
- Object.assign({}, expandedRowData[record.id],
- {loading: false, dataSource: containerKeysResponse.keys, totalCount: containerKeysResponse.totalCount});
- return {
- expandedRowData: Object.assign({}, expandedRowData, {[record.id]: expandedRowState})
- }
- });
- });
- }
- };
-
- expandedRowRender = (record: MissingContainerResponse) => {
- const {expandedRowData} = this.state;
- const containerId = record.id;
- if (expandedRowData[containerId]) {
- const containerKeys: ExpandedRowState = expandedRowData[containerId];
- const paginationConfig: PaginationConfig = {
- showTotal: (total: number, range) => `${range[0]}-${range[1]} of ${total} keys`
- };
- return <Table loading={containerKeys.loading} dataSource={containerKeys.dataSource}
- columns={KEY_TABLE_COLUMNS} pagination={paginationConfig}/>
- }
- return <div>Loading...</div>;
- };
-
- render () {
- const {dataSource, loading, totalCount} = this.state;
- const paginationConfig: PaginationConfig = {
- showTotal: (total: number, range) => `${range[0]}-${range[1]} of ${total} missing containers`,
- showSizeChanger: true,
- onShowSizeChange: this.onShowSizeChange
- };
- return (
- <div className="missing-containers-container">
- <div className="page-header">
- Missing Containers ({totalCount})
- </div>
- <div className="content-div">
- <Table dataSource={dataSource} columns={COLUMNS} loading={loading} pagination={paginationConfig}
- rowKey="id" expandedRowRender={this.expandedRowRender} expandRowByClick={true} onExpand={this.onRowExpandClick}/>
- </div>
- </div>
- );
- }
-}
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Overview/Overview.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Overview/Overview.tsx
deleted file mode 100644
index ddff61e..0000000
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Overview/Overview.tsx
+++ /dev/null
@@ -1,144 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import React from 'react';
-import {Row, Col, Icon, Tooltip} from 'antd';
-import OverviewCard from 'components/OverviewCard/OverviewCard';
-import axios from 'axios';
-import prettyBytes from 'pretty-bytes';
-import './Overview.less';
-import {StorageReport} from "types/datanode.types";
-
-interface ClusterStateResponse {
- totalDatanodes: number;
- healthyDatanodes: number;
- pipelines: number;
- storageReport: StorageReport;
- containers: number;
- volumes: number;
- buckets: number;
- keys: number;
-}
-
-interface OverviewState {
- loading: boolean;
- datanodes: string;
- pipelines: number;
- storageReport: StorageReport;
- containers: number;
- volumes: number;
- buckets: number;
- keys: number;
- missingContainersCount: number;
-}
-
-export class Overview extends React.Component<any, OverviewState> {
-
- constructor(props: any) {
- super(props);
- this.state = {
- loading: false,
- datanodes: '',
- pipelines: 0,
- storageReport: {
- capacity: 0,
- used: 0,
- remaining: 0
- },
- containers: 0,
- volumes: 0,
- buckets: 0,
- keys: 0,
- missingContainersCount: 0
- }
- }
-
- componentDidMount(): void {
- this.setState({
- loading: true
- });
- axios.all([
- axios.get('/api/v1/clusterState')
- ]).then(axios.spread((clusterStateResponse) => {
- const clusterState: ClusterStateResponse = clusterStateResponse.data;
- this.setState({
- loading: false,
- datanodes: `${clusterState.healthyDatanodes}/${clusterState.totalDatanodes}`,
- storageReport: clusterState.storageReport,
- pipelines: clusterState.pipelines,
- containers: clusterState.containers,
- volumes: clusterState.volumes,
- buckets: clusterState.buckets,
- keys: clusterState.keys,
- missingContainersCount: 0
- });
- }));
- }
-
- render() {
- const {loading, datanodes, pipelines, storageReport, containers, volumes, buckets,
- keys, missingContainersCount} = this.state;
- const datanodesElement = <span>
- <Icon type="check-circle" theme="filled" className="icon-success icon-small"/> {datanodes} <span className="ant-card-meta-description meta">HEALTHY</span>
- </span>;
- const containersTooltip = missingContainersCount === 1 ? "container is missing" : "containers are missing";
- const containersLink = missingContainersCount > 0 ? '/MissingContainers' : '';
- const containersElement = missingContainersCount > 0 ?
- <span>
- <Tooltip placement="bottom" title={`${missingContainersCount} ${containersTooltip}`}>
- <Icon type="exclamation-circle" theme="filled" className="icon-failure icon-small"/>
- </Tooltip>
- <span className="padded-text">{containers}</span>
- </span>
- : containers.toString();
- const clusterCapacity = `${prettyBytes(storageReport.capacity - storageReport.remaining)}/${prettyBytes(storageReport.capacity)}`;
- return (
- <div className="overview-content">
- <Row gutter={[25, 25]}>
- <Col xs={24} sm={18} md={12} lg={12} xl={6}>
- <OverviewCard loading={loading} title={"Datanodes"} data={datanodesElement} icon={"cluster"} hoverable={true}
- linkToUrl="/Datanodes"/>
- </Col>
- <Col xs={24} sm={18} md={12} lg={12} xl={6}>
- <OverviewCard loading={loading} title={"Pipelines"} data={pipelines.toString()} icon={"deployment-unit"} hoverable={true}
- linkToUrl="/Pipelines"/>
- </Col>
- <Col xs={24} sm={18} md={12} lg={12} xl={6}>
- <OverviewCard loading={loading} title={"Cluster Capacity"} data={clusterCapacity} icon={"database"}
- storageReport={storageReport}/>
- </Col>
- <Col xs={24} sm={18} md={12} lg={12} xl={6}>
- <OverviewCard loading={loading} title={"Containers"} data={containersElement} icon={"container"}
- error={missingContainersCount > 0} linkToUrl={containersLink}/>
- </Col>
- </Row>
- <Row gutter={[25, 25]}>
- <Col xs={24} sm={18} md={12} lg={12} xl={6}>
- <OverviewCard loading={loading} title={"Volumes"} data={volumes.toString()} icon={"inbox"}/>
- </Col>
- <Col xs={24} sm={18} md={12} lg={12} xl={6}>
- <OverviewCard loading={loading} title={"Buckets"} data={buckets.toString()} icon={"folder-open"}/>
- </Col>
- <Col xs={24} sm={18} md={12} lg={12} xl={6}>
- <OverviewCard loading={loading} title={"Keys"} data={keys.toString()} icon={"file-text"}/>
- </Col>
- </Row>
- </div>
- );
- }
-}
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Pipelines/Pipelines.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Pipelines/Pipelines.tsx
deleted file mode 100644
index f94928c..0000000
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Pipelines/Pipelines.tsx
+++ /dev/null
@@ -1,197 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import React from 'react';
-import axios from 'axios';
-import {Table, Tabs} from 'antd';
-import './Pipelines.less';
-import {PaginationConfig} from "antd/lib/pagination";
-import prettyMilliseconds from "pretty-ms";
-import moment from 'moment';
-import {ReplicationIcon} from "../../utils/themeIcons";
-const {TabPane} = Tabs;
-
-export type PipelineStatus = "active" | "inactive";
-
-interface PipelineResponse {
- pipelineId: string;
- status: PipelineStatus;
- replicationType: string;
- leaderNode: string;
- datanodes: string[];
- lastLeaderElection: number;
- duration: number;
- leaderElections: number;
- replicationFactor: number;
- containers: number;
-}
-
-interface PipelinesResponse {
- totalCount: number;
- pipelines: PipelineResponse[];
-}
-
-interface PipelinesState {
- activeLoading: boolean;
- activeDataSource: PipelineResponse[];
- activeTotalCount: number;
- inactiveLoading: boolean;
- inactiveDataSource: PipelineResponse[];
- inactiveTotalCount: number;
-}
-
-const COLUMNS = [
- {
- title: 'Pipeline ID',
- dataIndex: 'pipelineId',
- key: 'pipelineId',
- sorter: (a: PipelineResponse, b: PipelineResponse) => a.pipelineId.localeCompare(b.pipelineId)
- },
- {
- title: 'Replication Type & Factor',
- dataIndex: 'replicationType',
- key: 'replicationType',
- render: (replicationType: string, record: PipelineResponse) => {
- const replicationFactor = record.replicationFactor;
- return (
- <span>
- <ReplicationIcon replicationFactor={replicationFactor} replicationType={replicationType}/>
- {replicationType} ({replicationFactor})
- </span>
- )
- },
- sorter: (a: PipelineResponse, b: PipelineResponse) =>
- (a.replicationType + a.replicationFactor).localeCompare(b.replicationType + b.replicationFactor),
- defaultSortOrder: 'descend' as const
- },
- {
- title: 'Status',
- dataIndex: 'status',
- key: 'status',
- sorter: (a: PipelineResponse, b: PipelineResponse) => a.status.localeCompare(b.status)
- },
- {
- title: 'Containers',
- dataIndex: 'containers',
- key: 'containers',
- sorter: (a: PipelineResponse, b: PipelineResponse) => a.containers - b.containers
- },
- {
- title: 'Datanodes',
- dataIndex: 'datanodes',
- key: 'datanodes',
- render: (datanodes: string[]) => <div>{datanodes.map(datanode => <div key={datanode}>{datanode}</div>)}</div>
- },
- {
- title: 'Leader',
- dataIndex: 'leaderNode',
- key: 'leaderNode',
- sorter: (a: PipelineResponse, b: PipelineResponse) => a.leaderNode.localeCompare(b.leaderNode)
- },
- {
- title: 'Last Leader Election',
- dataIndex: 'lastLeaderElection',
- key: 'lastLeaderElection',
- render: (lastLeaderElection: number) => lastLeaderElection > 0 ?
- moment(lastLeaderElection).format('lll') : 'NA',
- sorter: (a: PipelineResponse, b: PipelineResponse) => a.lastLeaderElection - b.lastLeaderElection
- },
- {
- title: 'Lifetime',
- dataIndex: 'duration',
- key: 'duration',
- render: (duration: number) => prettyMilliseconds(duration, {compact: true}),
- sorter: (a: PipelineResponse, b: PipelineResponse) => a.duration - b.duration
- },
- {
- title: 'No. of Elections',
- dataIndex: 'leaderElections',
- key: 'leaderElections',
- sorter: (a: PipelineResponse, b: PipelineResponse) => a.leaderElections - b.leaderElections
- }
-];
-
-export class Pipelines extends React.Component<any, PipelinesState> {
-
- constructor(props: any) {
- super(props);
- this.state = {
- activeLoading: false,
- activeDataSource: [],
- activeTotalCount: 0,
- inactiveLoading: false,
- inactiveDataSource: [],
- inactiveTotalCount: 0
- }
- }
-
- componentDidMount(): void {
- // Fetch pipelines on component mount
- this.setState({
- activeLoading: true
- });
- axios.get('/api/v1/pipelines').then(response => {
- const pipelinesResponse: PipelinesResponse = response.data;
- const totalCount = pipelinesResponse.totalCount;
- const pipelines: PipelineResponse[] = pipelinesResponse.pipelines;
- this.setState({
- activeLoading: false,
- activeDataSource: pipelines,
- activeTotalCount: totalCount
- });
- });
- }
-
- onShowSizeChange = (current: number, pageSize: number) => {
- // TODO: Implement this method once server side pagination is enabled
- console.log(current, pageSize);
- };
-
- onTabChange = (activeKey: string) => {
- // Fetch inactive pipelines if tab is switched to "Inactive"
- if (activeKey === "2") {
- // TODO: Trigger an ajax request to fetch inactive pipelines
- }
- };
-
- render () {
- const {activeDataSource, activeLoading, activeTotalCount} = this.state;
- const paginationConfig: PaginationConfig = {
- showTotal: (total: number, range) => `${range[0]}-${range[1]} of ${total} pipelines`,
- showSizeChanger: true,
- onShowSizeChange: this.onShowSizeChange
- };
- return (
- <div className="pipelines-container">
- <div className="page-header">
- Pipelines ({activeTotalCount})
- </div>
- <div className="content-div">
- <Tabs defaultActiveKey="1" onChange={this.onTabChange}>
- <TabPane key="1" tab="Active">
- <Table dataSource={activeDataSource} columns={COLUMNS} loading={activeLoading} pagination={paginationConfig} rowKey="pipelineId"/>
- </TabPane>
- <TabPane key="2" tab="Inactive">
-
- </TabPane>
- </Tabs>
- </div>
- </div>
- );
- }
-}
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Datanodes/Datanodes.less b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/datanodes/datanodes.less
similarity index 99%
rename from hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Datanodes/Datanodes.less
rename to hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/datanodes/datanodes.less
index 1520e9e..4a3cdf5 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Datanodes/Datanodes.less
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/datanodes/datanodes.less
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/datanodes/datanodes.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/datanodes/datanodes.tsx
new file mode 100644
index 0000000..feb5b6f
--- /dev/null
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/datanodes/datanodes.tsx
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import axios from 'axios';
+import {Table, Icon, Tooltip} from 'antd';
+import {PaginationConfig} from 'antd/lib/pagination';
+import moment from 'moment';
+import {ReplicationIcon} from 'utils/themeIcons';
+import StorageBar from 'components/storageBar/storageBar';
+import {DatanodeStatus, IStorageReport} from 'types/datanode.types';
+import './datanodes.less';
+import {AutoReloadHelper} from 'utils/autoReloadHelper';
+import AutoReloadPanel from 'components/autoReloadPanel/autoReloadPanel';
+import {showDataFetchError} from 'utils/common';
+
+interface IDatanodeResponse {
+ hostname: string;
+ state: DatanodeStatus;
+ lastHeartbeat: number;
+ storageReport: IStorageReport;
+ pipelines: IPipeline[];
+ containers: number;
+ leaderCount: number;
+}
+
+interface IDatanodesResponse {
+ totalCount: number;
+ datanodes: IDatanodeResponse[];
+}
+
+interface IDatanode {
+ hostname: string;
+ state: DatanodeStatus;
+ lastHeartbeat: number;
+ storageUsed: number;
+ storageTotal: number;
+ storageRemaining: number;
+ pipelines: IPipeline[];
+ containers: number;
+ leaderCount: number;
+}
+
+interface IPipeline {
+ pipelineID: string;
+ replicationType: string;
+ replicationFactor: number;
+ leaderNode: string;
+}
+
+interface IDatanodesState {
+ loading: boolean;
+ dataSource: IDatanode[];
+ totalCount: number;
+ lastUpdated: number;
+}
+
+const renderDatanodeStatus = (status: DatanodeStatus) => {
+ const statusIconMap = {
+ HEALTHY: <Icon type='check-circle' theme='filled' twoToneColor='#1da57a' className='icon-success'/>,
+ STALE: <Icon type='hourglass' theme='filled' className='icon-warning'/>,
+ DEAD: <Icon type='close-circle' theme='filled' className='icon-failure'/>,
+ DECOMMISSIONING: <Icon type='warning' theme='filled' className='icon-warning'/>,
+ DECOMMISSIONED: <Icon type='exclamation-circle' theme='filled' className='icon-failure'/>
+ };
+ const icon = status in statusIconMap ? statusIconMap[status] : '';
+ return <span>{icon} {status}</span>;
+};
+
+const COLUMNS = [
+ {
+ title: 'Status',
+ dataIndex: 'state',
+ key: 'state',
+ render: (text: DatanodeStatus) => renderDatanodeStatus(text),
+ sorter: (a: IDatanode, b: IDatanode) => a.state.localeCompare(b.state)
+ },
+ {
+ title: 'Hostname',
+ dataIndex: 'hostname',
+ key: 'hostname',
+ sorter: (a: IDatanode, b: IDatanode) => a.hostname.localeCompare(b.hostname),
+ defaultSortOrder: 'ascend' as const
+ },
+ {
+ title: 'Storage Capacity',
+ dataIndex: 'storageUsed',
+ key: 'storageUsed',
+ sorter: (a: IDatanode, b: IDatanode) => a.storageRemaining - b.storageRemaining,
+ render: (text: string, record: IDatanode) => (
+ <StorageBar
+ total={record.storageTotal} used={record.storageUsed}
+ remaining={record.storageRemaining}/>
+ )},
+ {
+ title: 'Last Heartbeat',
+ dataIndex: 'lastHeartbeat',
+ key: 'lastHeartbeat',
+ sorter: (a: IDatanode, b: IDatanode) => a.lastHeartbeat - b.lastHeartbeat,
+ render: (heartbeat: number) => {
+ return heartbeat > 0 ? moment(heartbeat).format('lll') : 'NA';
+ }
+ },
+ {
+ title: 'Pipeline ID(s)',
+ dataIndex: 'pipelines',
+ key: 'pipelines',
+ render: (pipelines: IPipeline[], record: IDatanode) => {
+ return (
+ <div>
+ {
+ pipelines.map((pipeline, index) => (
+ <div key={index} className='pipeline-container'>
+ <ReplicationIcon replicationFactor={pipeline.replicationFactor}
+ replicationType={pipeline.replicationType}
+ leaderNode={pipeline.leaderNode}
+ isLeader={pipeline.leaderNode === record.hostname}/>
+ {pipeline.pipelineID}
+ </div>
+ ))
+ }
+ </div>
+ );
+ }
+ },
+ {
+ title: <span>
+ Leader Count
+ <Tooltip title='The number of Ratis Pipelines in which the given datanode is elected as a leader.'>
+ <Icon type='info-circle'/>
+ </Tooltip>
+ </span>,
+ dataIndex: 'leaderCount',
+ key: 'leaderCount',
+ sorter: (a: IDatanode, b: IDatanode) => a.leaderCount - b.leaderCount
+ },
+ {
+ title: 'Containers',
+ dataIndex: 'containers',
+ key: 'containers',
+ sorter: (a: IDatanode, b: IDatanode) => a.containers - b.containers
+ }
+];
+
+export class Datanodes extends React.Component<Record<string, object>, IDatanodesState> {
+ autoReload: AutoReloadHelper;
+
+ constructor(props = {}) {
+ super(props);
+ this.state = {
+ loading: false,
+ dataSource: [],
+ totalCount: 0,
+ lastUpdated: 0
+ };
+ this.autoReload = new AutoReloadHelper(this._loadData);
+ }
+
+ _loadData = () => {
+ this.setState({
+ loading: true
+ });
+ axios.get('/api/v1/datanodes').then(response => {
+ const datanodesResponse: IDatanodesResponse = response.data;
+ const totalCount = datanodesResponse.totalCount;
+ const datanodes: IDatanodeResponse[] = datanodesResponse.datanodes;
+ const dataSource: IDatanode[] = datanodes.map(datanode => {
+ return {
+ hostname: datanode.hostname,
+ state: datanode.state,
+ lastHeartbeat: datanode.lastHeartbeat,
+ storageUsed: datanode.storageReport.used,
+ storageTotal: datanode.storageReport.capacity,
+ storageRemaining: datanode.storageReport.remaining,
+ pipelines: datanode.pipelines,
+ containers: datanode.containers,
+ leaderCount: datanode.leaderCount
+ };
+ });
+ this.setState({
+ loading: false,
+ dataSource,
+ totalCount,
+ lastUpdated: Number(moment())
+ });
+ }).catch(error => {
+ this.setState({
+ loading: false
+ });
+ showDataFetchError(error.toString());
+ });
+ };
+
+ componentDidMount(): void {
+ // Fetch datanodes on component mount
+ this._loadData();
+ this.autoReload.startPolling();
+ }
+
+ componentWillUnmount(): void {
+ this.autoReload.stopPolling();
+ }
+
+ onShowSizeChange = (current: number, pageSize: number) => {
+ console.log(current, pageSize);
+ };
+
+ render() {
+ const {dataSource, loading, totalCount, lastUpdated} = this.state;
+ const paginationConfig: PaginationConfig = {
+ showTotal: (total: number, range) => `${range[0]}-${range[1]} of ${total} datanodes`,
+ showSizeChanger: true,
+ onShowSizeChange: this.onShowSizeChange
+ };
+ return (
+ <div className='datanodes-container'>
+ <div className='page-header'>
+ Datanodes ({totalCount})
+ <AutoReloadPanel isLoading={loading} lastUpdated={lastUpdated} togglePolling={this.autoReload.handleAutoReloadToggle} onReload={this._loadData}/>
+ </div>
+ <div className='content-div'>
+ <Table dataSource={dataSource} columns={COLUMNS} loading={loading} pagination={paginationConfig} rowKey='hostname'/>
+ </div>
+ </div>
+ );
+ }
+}
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/MissingContainers/MissingContainers.less b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/missingContainers/missingContainers.less
similarity index 95%
rename from hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/MissingContainers/MissingContainers.less
rename to hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/missingContainers/missingContainers.less
index 5d5f1e3..66a368a 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/MissingContainers/MissingContainers.less
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/missingContainers/missingContainers.less
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -15,3 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
+.pl-5 {
+ padding-left: 5px;
+}
\ No newline at end of file
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/missingContainers/missingContainers.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/missingContainers/missingContainers.tsx
new file mode 100644
index 0000000..bc87483
--- /dev/null
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/missingContainers/missingContainers.tsx
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import axios from 'axios';
+import {Icon, Table, Tooltip} from 'antd';
+import './missingContainers.less';
+import {PaginationConfig} from 'antd/lib/pagination';
+import prettyBytes from 'pretty-bytes';
+import moment from 'moment';
+import {showDataFetchError, timeFormat} from '../../utils/common';
+
+interface IMissingContainerResponse {
+ containerID: number;
+ keys: number;
+ replicas: IContainerReplica[];
+ missingSince: number;
+ pipelineID: string;
+}
+
+export interface IContainerReplica {
+ containerId: number;
+ datanodeHost: string;
+ firstReportTimestamp: number;
+ lastReportTimestamp: number;
+}
+
+export interface IMissingContainersResponse {
+ totalCount: number;
+ containers: IMissingContainerResponse[];
+}
+
+interface IKeyResponse {
+ Volume: string;
+ Bucket: string;
+ Key: string;
+ DataSize: number;
+ Versions: number[];
+ Blocks: object;
+ CreationTime: string;
+ ModificationTime: string;
+}
+
+interface IContainerKeysResponse {
+ totalCount: number;
+ keys: IKeyResponse[];
+}
+
+const COLUMNS = [
+ {
+ title: 'Container ID',
+ dataIndex: 'containerID',
+ key: 'containerID',
+ sorter: (a: IMissingContainerResponse, b: IMissingContainerResponse) => a.containerID - b.containerID
+ },
+ {
+ title: 'No. of Keys',
+ dataIndex: 'keys',
+ key: 'keys',
+ sorter: (a: IMissingContainerResponse, b: IMissingContainerResponse) => a.keys - b.keys
+ },
+ {
+ title: 'Datanodes',
+ dataIndex: 'replicas',
+ key: 'replicas',
+ render: (replicas: IContainerReplica[]) => (
+ <div>
+ {replicas.map(replica => {
+ const tooltip = (
+ <div>
+ <div>First Report Time: {timeFormat(replica.firstReportTimestamp)}</div>
+ <div>Last Report Time: {timeFormat(replica.lastReportTimestamp)}</div>
+ </div>
+ );
+ return (
+ <div key={replica.datanodeHost}>
+ <Tooltip
+ placement='left'
+ title={tooltip}
+ >
+ <Icon type='info-circle' className='icon-small'/>
+ </Tooltip>
+ <span className='pl-5'>
+ {replica.datanodeHost}
+ </span>
+ </div>
+ );
+ }
+ )}
+ </div>
+ )
+ },
+ {
+ title: 'Pipeline ID',
+ dataIndex: 'pipelineID',
+ key: 'pipelineID',
+ sorter: (a: IMissingContainerResponse, b: IMissingContainerResponse) => a.pipelineID.localeCompare(b.pipelineID)
+ },
+ {
+ title: 'Missing Since',
+ dataIndex: 'missingSince',
+ key: 'missingSince',
+ render: (missingSince: number) => timeFormat(missingSince),
+ sorter: (a: IMissingContainerResponse, b: IMissingContainerResponse) => a.missingSince - b.missingSince
+ }
+];
+
+const KEY_TABLE_COLUMNS = [
+ {
+ title: 'Volume',
+ dataIndex: 'Volume',
+ key: 'Volume'
+ },
+ {
+ title: 'Bucket',
+ dataIndex: 'Bucket',
+ key: 'Bucket'
+ },
+ {
+ title: 'Key',
+ dataIndex: 'Key',
+ key: 'Key'
+ },
+ {
+ title: 'Size',
+ dataIndex: 'DataSize',
+ key: 'DataSize',
+ render: (dataSize: number) => <div>{prettyBytes(dataSize)}</div>
+ },
+ {
+ title: 'Date Created',
+ dataIndex: 'CreationTime',
+ key: 'CreationTime',
+ render: (date: string) => moment(date).format('lll')
+ },
+ {
+ title: 'Date Modified',
+ dataIndex: 'ModificationTime',
+ key: 'ModificationTime',
+ render: (date: string) => moment(date).format('lll')
+ }
+];
+
+interface IExpandedRow {
+ [key: number]: IExpandedRowState;
+}
+
+interface IExpandedRowState {
+ containerId: number;
+ loading: boolean;
+ dataSource: IKeyResponse[];
+ totalCount: number;
+}
+
+interface IMissingContainersState {
+ loading: boolean;
+ dataSource: IMissingContainerResponse[];
+ totalCount: number;
+ expandedRowData: IExpandedRow;
+}
+
+export class MissingContainers extends React.Component<Record<string, object>, IMissingContainersState> {
+ constructor(props = {}) {
+ super(props);
+ this.state = {
+ loading: false,
+ dataSource: [],
+ totalCount: 0,
+ expandedRowData: {}
+ };
+ }
+
+ componentDidMount(): void {
+ // Fetch missing containers on component mount
+ this.setState({
+ loading: true
+ });
+ axios.get('/api/v1/containers/missing').then(response => {
+ const missingContainersResponse: IMissingContainersResponse = response.data;
+ const totalCount = missingContainersResponse.totalCount;
+ const missingContainers: IMissingContainerResponse[] = missingContainersResponse.containers;
+ this.setState({
+ loading: false,
+ dataSource: missingContainers,
+ totalCount
+ });
+ }).catch(error => {
+ this.setState({
+ loading: false
+ });
+ showDataFetchError(error.toString());
+ });
+ }
+
+ onShowSizeChange = (current: number, pageSize: number) => {
+ console.log(current, pageSize);
+ };
+
+ onRowExpandClick = (expanded: boolean, record: IMissingContainerResponse) => {
+ if (expanded) {
+ this.setState(({expandedRowData}) => {
+ const expandedRowState: IExpandedRowState = expandedRowData[record.containerID] ?
+ Object.assign({}, expandedRowData[record.containerID], {loading: true}) :
+ {containerId: record.containerID, loading: true, dataSource: [], totalCount: 0};
+ return {
+ expandedRowData: Object.assign({}, expandedRowData, {[record.containerID]: expandedRowState})
+ };
+ });
+ axios.get(`/api/v1/containers/${record.containerID}/keys`).then(response => {
+ const containerKeysResponse: IContainerKeysResponse = response.data;
+ this.setState(({expandedRowData}) => {
+ const expandedRowState: IExpandedRowState =
+ Object.assign({}, expandedRowData[record.containerID],
+ {loading: false, dataSource: containerKeysResponse.keys, totalCount: containerKeysResponse.totalCount});
+ return {
+ expandedRowData: Object.assign({}, expandedRowData, {[record.containerID]: expandedRowState})
+ };
+ });
+ }).catch(error => {
+ this.setState(({expandedRowData}) => {
+ const expandedRowState: IExpandedRowState =
+ Object.assign({}, expandedRowData[record.containerID],
+ {loading: false});
+ return {
+ expandedRowData: Object.assign({}, expandedRowData, {[record.containerID]: expandedRowState})
+ };
+ });
+ showDataFetchError(error.toString());
+ });
+ }
+ };
+
+ expandedRowRender = (record: IMissingContainerResponse) => {
+ const {expandedRowData} = this.state;
+ const containerId = record.containerID;
+ if (expandedRowData[containerId]) {
+ const containerKeys: IExpandedRowState = expandedRowData[containerId];
+ const dataSource = containerKeys.dataSource.map(record => (
+ {...record, uid: `${record.Volume}/${record.Bucket}/${record.Key}`}
+ ));
+ const paginationConfig: PaginationConfig = {
+ showTotal: (total: number, range) => `${range[0]}-${range[1]} of ${total} keys`
+ };
+ return (
+ <Table
+ loading={containerKeys.loading} dataSource={dataSource}
+ columns={KEY_TABLE_COLUMNS} pagination={paginationConfig}
+ rowKey='uid'/>
+ );
+ }
+
+ return <div>Loading...</div>;
+ };
+
+ render() {
+ const {dataSource, loading, totalCount} = this.state;
+ const paginationConfig: PaginationConfig = {
+ showTotal: (total: number, range) => `${range[0]}-${range[1]} of ${total} missing containers`,
+ showSizeChanger: true,
+ onShowSizeChange: this.onShowSizeChange
+ };
+ return (
+ <div className='missing-containers-container'>
+ <div className='page-header'>
+ Missing Containers ({totalCount})
+ </div>
+ <div className='content-div'>
+ <Table
+ expandRowByClick dataSource={dataSource} columns={COLUMNS}
+ loading={loading}
+ pagination={paginationConfig} rowKey='containerID'
+ expandedRowRender={this.expandedRowRender} onExpand={this.onRowExpandClick}/>
+ </div>
+ </div>
+ );
+ }
+}
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/NotFound/NotFound.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/notFound/notFound.tsx
similarity index 89%
rename from hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/NotFound/NotFound.tsx
rename to hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/notFound/notFound.tsx
index cc6ac4a..4a2582b 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/NotFound/NotFound.tsx
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/notFound/notFound.tsx
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -20,10 +20,10 @@
export const NotFound: React.FC = () => {
return (
- <div>
- <div className="page-header">
- 404 Page Not Found :(
- </div>
+ <div>
+ <div className='page-header'>
+ 404 Page Not Found :(
</div>
+ </div>
);
};
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Overview/Overview.less b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/overview/overview.less
similarity index 97%
rename from hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Overview/Overview.less
rename to hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/overview/overview.less
index 5a8a069..3433707 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Overview/Overview.less
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/overview/overview.less
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -17,7 +17,7 @@
*/
.overview-content {
- margin: 20px 5px;
+ margin: 0 5px 20px;
.icon-small {
font-size: 16px;
}
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/overview/overview.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/overview/overview.tsx
new file mode 100644
index 0000000..7e84a0d
--- /dev/null
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/overview/overview.tsx
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import {Row, Col, Icon, Tooltip} from 'antd';
+import OverviewCard from 'components/overviewCard/overviewCard';
+import axios from 'axios';
+import prettyBytes from 'pretty-bytes';
+import './overview.less';
+import {IStorageReport} from 'types/datanode.types';
+import {IMissingContainersResponse} from '../missingContainers/missingContainers';
+import moment from 'moment';
+import AutoReloadPanel from 'components/autoReloadPanel/autoReloadPanel';
+import {showDataFetchError} from 'utils/common';
+import {AutoReloadHelper} from 'utils/autoReloadHelper';
+
+interface IClusterStateResponse {
+ totalDatanodes: number;
+ healthyDatanodes: number;
+ pipelines: number;
+ storageReport: IStorageReport;
+ containers: number;
+ volumes: number;
+ buckets: number;
+ keys: number;
+}
+
+interface IOverviewState {
+ loading: boolean;
+ datanodes: string;
+ pipelines: number;
+ storageReport: IStorageReport;
+ containers: number;
+ volumes: number;
+ buckets: number;
+ keys: number;
+ missingContainersCount: number;
+ lastUpdated: number;
+}
+
+export class Overview extends React.Component<Record<string, object>, IOverviewState> {
+ interval = 0;
+ autoReload: AutoReloadHelper;
+
+ constructor(props = {}) {
+ super(props);
+ this.state = {
+ loading: false,
+ datanodes: '',
+ pipelines: 0,
+ storageReport: {
+ capacity: 0,
+ used: 0,
+ remaining: 0
+ },
+ containers: 0,
+ volumes: 0,
+ buckets: 0,
+ keys: 0,
+ missingContainersCount: 0,
+ lastUpdated: 0
+ };
+ this.autoReload = new AutoReloadHelper(this._loadData);
+ }
+
+ _loadData = () => {
+ this.setState({
+ loading: true
+ });
+ axios.all([
+ axios.get('/api/v1/clusterState'),
+ axios.get('/api/v1/containers/missing')
+ ]).then(axios.spread((clusterStateResponse, missingContainersResponse) => {
+ const clusterState: IClusterStateResponse = clusterStateResponse.data;
+ const missingContainers: IMissingContainersResponse = missingContainersResponse.data;
+ const missingContainersCount = missingContainers.totalCount;
+ this.setState({
+ loading: false,
+ datanodes: `${clusterState.healthyDatanodes}/${clusterState.totalDatanodes}`,
+ storageReport: clusterState.storageReport,
+ pipelines: clusterState.pipelines,
+ containers: clusterState.containers,
+ volumes: clusterState.volumes,
+ buckets: clusterState.buckets,
+ keys: clusterState.keys,
+ missingContainersCount,
+ lastUpdated: Number(moment())
+ });
+ })).catch(error => {
+ this.setState({
+ loading: false
+ });
+ showDataFetchError(error.toString());
+ });
+ };
+
+ componentDidMount(): void {
+ this._loadData();
+ this.autoReload.startPolling();
+ }
+
+ componentWillUnmount(): void {
+ this.autoReload.stopPolling();
+ }
+
+ render() {
+ const {loading, datanodes, pipelines, storageReport, containers, volumes, buckets,
+ keys, missingContainersCount, lastUpdated} = this.state;
+ const datanodesElement = (
+ <span>
+ <Icon type='check-circle' theme='filled' className='icon-success icon-small'/> {datanodes} <span className='ant-card-meta-description meta'>HEALTHY</span>
+ </span>
+ );
+ const containersTooltip = missingContainersCount === 1 ? 'container is missing' : 'containers are missing';
+ const containersLink = missingContainersCount > 0 ? '/MissingContainers' : '';
+ const containersElement = missingContainersCount > 0 ? (
+ <span>
+ <Tooltip placement='bottom' title={`${missingContainersCount} ${containersTooltip}`}>
+ <Icon type='exclamation-circle' theme='filled' className='icon-failure icon-small'/>
+ </Tooltip>
+ <span className='padded-text'>{containers - missingContainersCount}/{containers}</span>
+ </span>
+ ) :
+ containers.toString();
+ const clusterCapacity = `${prettyBytes(storageReport.capacity - storageReport.remaining)}/${prettyBytes(storageReport.capacity)}`;
+ return (
+ <div className='overview-content'>
+ <div className='page-header'>
+ Overview
+ <AutoReloadPanel isLoading={loading} lastUpdated={lastUpdated} togglePolling={this.autoReload.handleAutoReloadToggle} onReload={this._loadData}/>
+ </div>
+ <Row gutter={[25, 25]}>
+ <Col xs={24} sm={18} md={12} lg={12} xl={6}>
+ <OverviewCard
+ hoverable loading={loading} title='Datanodes'
+ data={datanodesElement} icon='cluster'
+ linkToUrl='/Datanodes'/>
+ </Col>
+ <Col xs={24} sm={18} md={12} lg={12} xl={6}>
+ <OverviewCard
+ hoverable loading={loading} title='Pipelines'
+ data={pipelines.toString()} icon='deployment-unit'
+ linkToUrl='/Pipelines'/>
+ </Col>
+ <Col xs={24} sm={18} md={12} lg={12} xl={6}>
+ <OverviewCard
+ loading={loading} title='Cluster Capacity' data={clusterCapacity}
+ icon='database'
+ storageReport={storageReport}/>
+ </Col>
+ <Col xs={24} sm={18} md={12} lg={12} xl={6}>
+ <OverviewCard
+ loading={loading} title='Containers' data={containersElement}
+ icon='container'
+ error={missingContainersCount > 0} linkToUrl={containersLink}/>
+ </Col>
+ </Row>
+ <Row gutter={[25, 25]}>
+ <Col xs={24} sm={18} md={12} lg={12} xl={6}>
+ <OverviewCard loading={loading} title='Volumes' data={volumes.toString()} icon='inbox'/>
+ </Col>
+ <Col xs={24} sm={18} md={12} lg={12} xl={6}>
+ <OverviewCard loading={loading} title='Buckets' data={buckets.toString()} icon='folder-open'/>
+ </Col>
+ <Col xs={24} sm={18} md={12} lg={12} xl={6}>
+ <OverviewCard loading={loading} title='Keys' data={keys.toString()} icon='file-text'/>
+ </Col>
+ </Row>
+ </div>
+ );
+ }
+}
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Pipelines/Pipelines.less b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/pipelines/pipelines.less
similarity index 99%
rename from hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Pipelines/Pipelines.less
rename to hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/pipelines/pipelines.less
index 8edff71..dec0a10 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Pipelines/Pipelines.less
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/pipelines/pipelines.less
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/pipelines/pipelines.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/pipelines/pipelines.tsx
new file mode 100644
index 0000000..f339060
--- /dev/null
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/pipelines/pipelines.tsx
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import axios from 'axios';
+import {Table, Tabs} from 'antd';
+import './pipelines.less';
+import {PaginationConfig} from 'antd/lib/pagination';
+import prettyMilliseconds from 'pretty-ms';
+import moment from 'moment';
+import {ReplicationIcon} from 'utils/themeIcons';
+import {AutoReloadHelper} from 'utils/autoReloadHelper';
+import AutoReloadPanel from 'components/autoReloadPanel/autoReloadPanel';
+import {showDataFetchError} from 'utils/common';
+import {IAxiosResponse} from 'types/axios.types';
+
+const {TabPane} = Tabs;
+export type PipelineStatus = 'active' | 'inactive';
+
+interface IPipelineResponse {
+ pipelineId: string;
+ status: PipelineStatus;
+ replicationType: string;
+ leaderNode: string;
+ datanodes: string[];
+ lastLeaderElection: number;
+ duration: number;
+ leaderElections: number;
+ replicationFactor: number;
+ containers: number;
+}
+
+interface IPipelinesResponse {
+ totalCount: number;
+ pipelines: IPipelineResponse[];
+}
+
+interface IPipelinesState {
+ activeLoading: boolean;
+ activeDataSource: IPipelineResponse[];
+ activeTotalCount: number;
+ lastUpdated: number;
+}
+
+const COLUMNS = [
+ {
+ title: 'Pipeline ID',
+ dataIndex: 'pipelineId',
+ key: 'pipelineId',
+ sorter: (a: IPipelineResponse, b: IPipelineResponse) => a.pipelineId.localeCompare(b.pipelineId)
+ },
+ {
+ title: 'Replication Type & Factor',
+ dataIndex: 'replicationType',
+ key: 'replicationType',
+ render: (replicationType: string, record: IPipelineResponse) => {
+ const replicationFactor = record.replicationFactor;
+ return (
+ <span>
+ <ReplicationIcon replicationFactor={replicationFactor}
+ replicationType={replicationType}
+ leaderNode={record.leaderNode}
+ isLeader={false}/>
+ {replicationType} ({replicationFactor})
+ </span>
+ );
+ },
+ sorter: (a: IPipelineResponse, b: IPipelineResponse) =>
+ (a.replicationType + a.replicationFactor.toString()).localeCompare(b.replicationType + b.replicationFactor.toString()),
+ defaultSortOrder: 'descend' as const
+ },
+ {
+ title: 'Status',
+ dataIndex: 'status',
+ key: 'status',
+ sorter: (a: IPipelineResponse, b: IPipelineResponse) => a.status.localeCompare(b.status)
+ },
+ {
+ title: 'Containers',
+ dataIndex: 'containers',
+ key: 'containers',
+ sorter: (a: IPipelineResponse, b: IPipelineResponse) => a.containers - b.containers
+ },
+ {
+ title: 'Datanodes',
+ dataIndex: 'datanodes',
+ key: 'datanodes',
+ render: (datanodes: string[]) => <div>{datanodes.map(datanode => <div key={datanode}>{datanode}</div>)}</div>
+ },
+ {
+ title: 'Leader',
+ dataIndex: 'leaderNode',
+ key: 'leaderNode',
+ sorter: (a: IPipelineResponse, b: IPipelineResponse) => a.leaderNode.localeCompare(b.leaderNode)
+ },
+ {
+ title: 'Last Leader Election',
+ dataIndex: 'lastLeaderElection',
+ key: 'lastLeaderElection',
+ render: (lastLeaderElection: number) => lastLeaderElection > 0 ?
+ moment(lastLeaderElection).format('lll') : 'NA',
+ sorter: (a: IPipelineResponse, b: IPipelineResponse) => a.lastLeaderElection - b.lastLeaderElection
+ },
+ {
+ title: 'Lifetime',
+ dataIndex: 'duration',
+ key: 'duration',
+ render: (duration: number) => prettyMilliseconds(duration, {compact: true}),
+ sorter: (a: IPipelineResponse, b: IPipelineResponse) => a.duration - b.duration
+ },
+ {
+ title: 'No. of Elections',
+ dataIndex: 'leaderElections',
+ key: 'leaderElections',
+ sorter: (a: IPipelineResponse, b: IPipelineResponse) => a.leaderElections - b.leaderElections
+ }
+];
+
+export class Pipelines extends React.Component<Record<string, object>, IPipelinesState> {
+ autoReload: AutoReloadHelper;
+
+ constructor(props = {}) {
+ super(props);
+ this.state = {
+ activeLoading: false,
+ activeDataSource: [],
+ activeTotalCount: 0,
+ lastUpdated: 0
+ };
+ this.autoReload = new AutoReloadHelper(this._loadData);
+ }
+
+ _loadData = () => {
+ this.setState({
+ activeLoading: true
+ });
+ axios.get('/api/v1/pipelines').then((response: IAxiosResponse<IPipelinesResponse>) => {
+ const pipelinesResponse: IPipelinesResponse = response.data;
+ const totalCount = pipelinesResponse.totalCount;
+ const pipelines: IPipelineResponse[] = pipelinesResponse.pipelines;
+ this.setState({
+ activeLoading: false,
+ activeDataSource: pipelines,
+ activeTotalCount: totalCount,
+ lastUpdated: Number(moment())
+ });
+ }).catch(error => {
+ this.setState({
+ activeLoading: false
+ });
+ showDataFetchError(error.toString());
+ });
+ };
+
+ componentDidMount(): void {
+ // Fetch pipelines on component mount
+ this._loadData();
+ this.autoReload.startPolling();
+ }
+
+ componentWillUnmount(): void {
+ this.autoReload.stopPolling();
+ }
+
+ onShowSizeChange = (current: number, pageSize: number) => {
+ console.log(current, pageSize);
+ };
+
+ onTabChange = (activeKey: string) => {
+ // Fetch inactive pipelines if tab is switched to "Inactive"
+ if (activeKey === '2') {
+ // Fetch inactive pipelines in the future
+ }
+ };
+
+ render() {
+ const {activeDataSource, activeLoading, activeTotalCount, lastUpdated} = this.state;
+ const paginationConfig: PaginationConfig = {
+ showTotal: (total: number, range) => `${range[0]}-${range[1]} of ${total} pipelines`,
+ showSizeChanger: true,
+ onShowSizeChange: this.onShowSizeChange
+ };
+ return (
+ <div className='pipelines-container'>
+ <div className='page-header'>
+ Pipelines ({activeTotalCount})
+ <AutoReloadPanel isLoading={activeLoading} lastUpdated={lastUpdated} togglePolling={this.autoReload.handleAutoReloadToggle} onReload={this._loadData}/>
+ </div>
+ <div className='content-div'>
+ <Tabs defaultActiveKey='1' onChange={this.onTabChange}>
+ <TabPane key='1' tab='Active'>
+ <Table dataSource={activeDataSource} columns={COLUMNS} loading={activeLoading} pagination={paginationConfig} rowKey='pipelineId'/>
+ </TabPane>
+ <TabPane key='2' tab='Inactive'/>
+ </Tabs>
+ </div>
+ </div>
+ );
+ }
+}
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/yarn.lock b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/yarn.lock
index c54aad7..d7fb91a 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/yarn.lock
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/yarn.lock
@@ -30,34 +30,52 @@
resolved "https://registry.yarnpkg.com/@ant-design/icons/-/icons-2.1.1.tgz#7b9c08dffd4f5d41db667d9dbe5e0107d0bd9a4a"
integrity sha512-jCH+k2Vjlno4YWl6g535nHR09PwCEmTBKAG6VqF+rhkrSPRLfgpU2maagwbZPLjaHuU5Jd1DFQ2KJpQuI6uG8w==
-"@babel/code-frame@7.5.5", "@babel/code-frame@^7.0.0", "@babel/code-frame@^7.5.5":
+"@babel/code-frame@7.8.3", "@babel/code-frame@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.8.3.tgz#33e25903d7481181534e12ec0a25f16b6fcf419e"
+ integrity sha512-a9gxpmdXtZEInkCSHUJDLHZVBgb1QS0jhss4cPP93EW7s+uC5bikET2twEF3KV+7rDblJcmNvTR7VJejqd2C2g==
+ dependencies:
+ "@babel/highlight" "^7.8.3"
+
+"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.5.5":
version "7.5.5"
resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.5.5.tgz#bc0782f6d69f7b7d49531219699b988f669a8f9d"
integrity sha512-27d4lZoomVyo51VegxI20xZPuSHusqbQag/ztrBC7wegWoQ1nLREPVSKSW8byhTlzTKyNE4ifaTA6lCp7JjpFw==
dependencies:
"@babel/highlight" "^7.0.0"
-"@babel/core@7.4.3":
- version "7.4.3"
- resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.4.3.tgz#198d6d3af4567be3989550d97e068de94503074f"
- integrity sha512-oDpASqKFlbspQfzAE7yaeTmdljSH2ADIvBlb0RwbStltTuWa0+7CCI1fYVINNv9saHPa1W7oaKeuNuKj+RQCvA==
+"@babel/compat-data@^7.9.0", "@babel/compat-data@^7.9.6":
+ version "7.9.6"
+ resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.9.6.tgz#3f604c40e420131affe6f2c8052e9a275ae2049b"
+ integrity sha512-5QPTrNen2bm7RBc7dsOmcA5hbrS4O2Vhmk5XOL4zWW/zD/hV0iinpefDlkm+tBBy8kDtFaaeEvmAqt+nURAV2g==
dependencies:
- "@babel/code-frame" "^7.0.0"
- "@babel/generator" "^7.4.0"
- "@babel/helpers" "^7.4.3"
- "@babel/parser" "^7.4.3"
- "@babel/template" "^7.4.0"
- "@babel/traverse" "^7.4.3"
- "@babel/types" "^7.4.0"
- convert-source-map "^1.1.0"
+ browserslist "^4.11.1"
+ invariant "^2.2.4"
+ semver "^5.5.0"
+
+"@babel/core@7.9.0":
+ version "7.9.0"
+ resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.9.0.tgz#ac977b538b77e132ff706f3b8a4dbad09c03c56e"
+ integrity sha512-kWc7L0fw1xwvI0zi8OKVBuxRVefwGOrKSQMvrQ3dW+bIIavBY3/NpXmpjMy7bQnLgwgzWQZ8TlM57YHpHNHz4w==
+ dependencies:
+ "@babel/code-frame" "^7.8.3"
+ "@babel/generator" "^7.9.0"
+ "@babel/helper-module-transforms" "^7.9.0"
+ "@babel/helpers" "^7.9.0"
+ "@babel/parser" "^7.9.0"
+ "@babel/template" "^7.8.6"
+ "@babel/traverse" "^7.9.0"
+ "@babel/types" "^7.9.0"
+ convert-source-map "^1.7.0"
debug "^4.1.0"
- json5 "^2.1.0"
- lodash "^4.17.11"
+ gensync "^1.0.0-beta.1"
+ json5 "^2.1.2"
+ lodash "^4.17.13"
resolve "^1.3.2"
semver "^5.4.1"
source-map "^0.5.0"
-"@babel/core@^7.0.0", "@babel/core@^7.1.0", "@babel/core@^7.1.6", "@babel/core@^7.4.5":
+"@babel/core@^7.0.0", "@babel/core@^7.1.0", "@babel/core@^7.4.5":
version "7.7.7"
resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.7.7.tgz#ee155d2e12300bcc0cff6a8ad46f2af5063803e9"
integrity sha512-jlSjuj/7z138NLZALxVgrx13AOtqip42ATZP7+kYl53GvDV6+4dCek1mVUo8z8c8Xnw/mx2q3d9HWh3griuesQ==
@@ -87,20 +105,46 @@
lodash "^4.17.13"
source-map "^0.5.0"
-"@babel/helper-annotate-as-pure@^7.0.0", "@babel/helper-annotate-as-pure@^7.7.4":
+"@babel/generator@^7.9.0", "@babel/generator@^7.9.6":
+ version "7.9.6"
+ resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.9.6.tgz#5408c82ac5de98cda0d77d8124e99fa1f2170a43"
+ integrity sha512-+htwWKJbH2bL72HRluF8zumBxzuX0ZZUFl3JLNyoUjM/Ho8wnVpPXM6aUz8cfKDqQ/h7zHqKt4xzJteUosckqQ==
+ dependencies:
+ "@babel/types" "^7.9.6"
+ jsesc "^2.5.1"
+ lodash "^4.17.13"
+ source-map "^0.5.0"
+
+"@babel/helper-annotate-as-pure@^7.7.4":
version "7.7.4"
resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.7.4.tgz#bb3faf1e74b74bd547e867e48f551fa6b098b6ce"
integrity sha512-2BQmQgECKzYKFPpiycoF9tlb5HA4lrVyAmLLVK177EcQAqjVLciUb2/R+n1boQ9y5ENV3uz2ZqiNw7QMBBw1Og==
dependencies:
"@babel/types" "^7.7.4"
-"@babel/helper-builder-binary-assignment-operator-visitor@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/helper-builder-binary-assignment-operator-visitor/-/helper-builder-binary-assignment-operator-visitor-7.7.4.tgz#5f73f2b28580e224b5b9bd03146a4015d6217f5f"
- integrity sha512-Biq/d/WtvfftWZ9Uf39hbPBYDUo986m5Bb4zhkeYDGUllF43D+nUe5M6Vuo6/8JDK/0YX/uBdeoQpyaNhNugZQ==
+"@babel/helper-annotate-as-pure@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.8.3.tgz#60bc0bc657f63a0924ff9a4b4a0b24a13cf4deee"
+ integrity sha512-6o+mJrZBxOoEX77Ezv9zwW7WV8DdluouRKNY/IR5u/YTMuKHgugHOzYWlYvYLpLA9nPsQCAAASpCIbjI9Mv+Uw==
dependencies:
- "@babel/helper-explode-assignable-expression" "^7.7.4"
- "@babel/types" "^7.7.4"
+ "@babel/types" "^7.8.3"
+
+"@babel/helper-builder-binary-assignment-operator-visitor@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/helper-builder-binary-assignment-operator-visitor/-/helper-builder-binary-assignment-operator-visitor-7.8.3.tgz#c84097a427a061ac56a1c30ebf54b7b22d241503"
+ integrity sha512-5eFOm2SyFPK4Rh3XMMRDjN7lBH0orh3ss0g3rTYZnBQ+r6YPj7lgDyCvPphynHvUrobJmeMignBr6Acw9mAPlw==
+ dependencies:
+ "@babel/helper-explode-assignable-expression" "^7.8.3"
+ "@babel/types" "^7.8.3"
+
+"@babel/helper-builder-react-jsx-experimental@^7.9.0":
+ version "7.9.5"
+ resolved "https://registry.yarnpkg.com/@babel/helper-builder-react-jsx-experimental/-/helper-builder-react-jsx-experimental-7.9.5.tgz#0b4b3e04e6123f03b404ca4dfd6528fe6bb92fe3"
+ integrity sha512-HAagjAC93tk748jcXpZ7oYRZH485RCq/+yEv9SIWezHRPv9moZArTnkUNciUNzvwHUABmiWKlcxJvMcu59UwTg==
+ dependencies:
+ "@babel/helper-annotate-as-pure" "^7.8.3"
+ "@babel/helper-module-imports" "^7.8.3"
+ "@babel/types" "^7.9.5"
"@babel/helper-builder-react-jsx@^7.7.4":
version "7.7.4"
@@ -110,53 +154,64 @@
"@babel/types" "^7.7.4"
esutils "^2.0.0"
-"@babel/helper-call-delegate@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/helper-call-delegate/-/helper-call-delegate-7.7.4.tgz#621b83e596722b50c0066f9dc37d3232e461b801"
- integrity sha512-8JH9/B7J7tCYJ2PpWVpw9JhPuEVHztagNVuQAFBVFYluRMlpG7F1CgKEgGeL6KFqcsIa92ZYVj6DSc0XwmN1ZA==
+"@babel/helper-builder-react-jsx@^7.9.0":
+ version "7.9.0"
+ resolved "https://registry.yarnpkg.com/@babel/helper-builder-react-jsx/-/helper-builder-react-jsx-7.9.0.tgz#16bf391990b57732700a3278d4d9a81231ea8d32"
+ integrity sha512-weiIo4gaoGgnhff54GQ3P5wsUQmnSwpkvU0r6ZHq6TzoSzKy4JxHEgnxNytaKbov2a9z/CVNyzliuCOUPEX3Jw==
dependencies:
- "@babel/helper-hoist-variables" "^7.7.4"
- "@babel/traverse" "^7.7.4"
- "@babel/types" "^7.7.4"
+ "@babel/helper-annotate-as-pure" "^7.8.3"
+ "@babel/types" "^7.9.0"
-"@babel/helper-create-class-features-plugin@^7.4.0", "@babel/helper-create-class-features-plugin@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.7.4.tgz#fce60939fd50618610942320a8d951b3b639da2d"
- integrity sha512-l+OnKACG4uiDHQ/aJT8dwpR+LhCJALxL0mJ6nzjB25e5IPwqV1VOsY7ah6UB1DG+VOXAIMtuC54rFJGiHkxjgA==
+"@babel/helper-compilation-targets@^7.8.7", "@babel/helper-compilation-targets@^7.9.6":
+ version "7.9.6"
+ resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.9.6.tgz#1e05b7ccc9d38d2f8b40b458b380a04dcfadd38a"
+ integrity sha512-x2Nvu0igO0ejXzx09B/1fGBxY9NXQlBW2kZsSxCJft+KHN8t9XWzIvFxtPHnBOAXpVsdxZKZFbRUC8TsNKajMw==
dependencies:
- "@babel/helper-function-name" "^7.7.4"
- "@babel/helper-member-expression-to-functions" "^7.7.4"
- "@babel/helper-optimise-call-expression" "^7.7.4"
- "@babel/helper-plugin-utils" "^7.0.0"
- "@babel/helper-replace-supers" "^7.7.4"
- "@babel/helper-split-export-declaration" "^7.7.4"
+ "@babel/compat-data" "^7.9.6"
+ browserslist "^4.11.1"
+ invariant "^2.2.4"
+ levenary "^1.1.1"
+ semver "^5.5.0"
-"@babel/helper-create-regexp-features-plugin@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.7.4.tgz#6d5762359fd34f4da1500e4cff9955b5299aaf59"
- integrity sha512-Mt+jBKaxL0zfOIWrfQpnfYCN7/rS6GKx6CCCfuoqVVd+17R8zNDlzVYmIi9qyb2wOk002NsmSTDymkIygDUH7A==
+"@babel/helper-create-class-features-plugin@^7.8.3", "@babel/helper-create-class-features-plugin@^7.9.6":
+ version "7.9.6"
+ resolved "https://registry.yarnpkg.com/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.9.6.tgz#965c8b0a9f051801fd9d3b372ca0ccf200a90897"
+ integrity sha512-6N9IeuyHvMBRyjNYOMJHrhwtu4WJMrYf8hVbEHD3pbbbmNOk1kmXSQs7bA4dYDUaIx4ZEzdnvo6NwC3WHd/Qow==
dependencies:
- "@babel/helper-regex" "^7.4.4"
- regexpu-core "^4.6.0"
+ "@babel/helper-function-name" "^7.9.5"
+ "@babel/helper-member-expression-to-functions" "^7.8.3"
+ "@babel/helper-optimise-call-expression" "^7.8.3"
+ "@babel/helper-plugin-utils" "^7.8.3"
+ "@babel/helper-replace-supers" "^7.9.6"
+ "@babel/helper-split-export-declaration" "^7.8.3"
-"@babel/helper-define-map@^7.4.0", "@babel/helper-define-map@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/helper-define-map/-/helper-define-map-7.7.4.tgz#2841bf92eb8bd9c906851546fe6b9d45e162f176"
- integrity sha512-v5LorqOa0nVQUvAUTUF3KPastvUt/HzByXNamKQ6RdJRTV7j8rLL+WB5C/MzzWAwOomxDhYFb1wLLxHqox86lg==
+"@babel/helper-create-regexp-features-plugin@^7.8.3", "@babel/helper-create-regexp-features-plugin@^7.8.8":
+ version "7.8.8"
+ resolved "https://registry.yarnpkg.com/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.8.8.tgz#5d84180b588f560b7864efaeea89243e58312087"
+ integrity sha512-LYVPdwkrQEiX9+1R29Ld/wTrmQu1SSKYnuOk3g0CkcZMA1p0gsNxJFj/3gBdaJ7Cg0Fnek5z0DsMULePP7Lrqg==
dependencies:
- "@babel/helper-function-name" "^7.7.4"
- "@babel/types" "^7.7.4"
+ "@babel/helper-annotate-as-pure" "^7.8.3"
+ "@babel/helper-regex" "^7.8.3"
+ regexpu-core "^4.7.0"
+
+"@babel/helper-define-map@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/helper-define-map/-/helper-define-map-7.8.3.tgz#a0655cad5451c3760b726eba875f1cd8faa02c15"
+ integrity sha512-PoeBYtxoZGtct3md6xZOCWPcKuMuk3IHhgxsRRNtnNShebf4C8YonTSblsK4tvDbm+eJAw2HAPOfCr+Q/YRG/g==
+ dependencies:
+ "@babel/helper-function-name" "^7.8.3"
+ "@babel/types" "^7.8.3"
lodash "^4.17.13"
-"@babel/helper-explode-assignable-expression@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.7.4.tgz#fa700878e008d85dc51ba43e9fb835cddfe05c84"
- integrity sha512-2/SicuFrNSXsZNBxe5UGdLr+HZg+raWBLE9vC98bdYOKX/U6PY0mdGlYUJdtTDPSU0Lw0PNbKKDpwYHJLn2jLg==
+"@babel/helper-explode-assignable-expression@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.8.3.tgz#a728dc5b4e89e30fc2dfc7d04fa28a930653f982"
+ integrity sha512-N+8eW86/Kj147bO9G2uclsg5pwfs/fqqY5rwgIL7eTBklgXjcOJ3btzS5iM6AitJcftnY7pm2lGsrJVYLGjzIw==
dependencies:
- "@babel/traverse" "^7.7.4"
- "@babel/types" "^7.7.4"
+ "@babel/traverse" "^7.8.3"
+ "@babel/types" "^7.8.3"
-"@babel/helper-function-name@^7.1.0", "@babel/helper-function-name@^7.7.4":
+"@babel/helper-function-name@^7.7.4":
version "7.7.4"
resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.7.4.tgz#ab6e041e7135d436d8f0a3eca15de5b67a341a2e"
integrity sha512-AnkGIdiBhEuiwdoMnKm7jfPfqItZhgRaZfMg1XX3bS25INOnLPjPG1Ppnajh8eqgt5kPJnfqrRHqFqmjKDZLzQ==
@@ -165,6 +220,15 @@
"@babel/template" "^7.7.4"
"@babel/types" "^7.7.4"
+"@babel/helper-function-name@^7.8.3", "@babel/helper-function-name@^7.9.5":
+ version "7.9.5"
+ resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.9.5.tgz#2b53820d35275120e1874a82e5aabe1376920a5c"
+ integrity sha512-JVcQZeXM59Cd1qanDUxv9fgJpt3NeKUaqBqUEvfmQ+BCOKq2xUgaWZW2hr0dkbyJgezYuplEoh5knmrnS68efw==
+ dependencies:
+ "@babel/helper-get-function-arity" "^7.8.3"
+ "@babel/template" "^7.8.3"
+ "@babel/types" "^7.9.5"
+
"@babel/helper-get-function-arity@^7.7.4":
version "7.7.4"
resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.7.4.tgz#cb46348d2f8808e632f0ab048172130e636005f0"
@@ -172,105 +236,137 @@
dependencies:
"@babel/types" "^7.7.4"
-"@babel/helper-hoist-variables@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.7.4.tgz#612384e3d823fdfaaf9fce31550fe5d4db0f3d12"
- integrity sha512-wQC4xyvc1Jo/FnLirL6CEgPgPCa8M74tOdjWpRhQYapz5JC7u3NYU1zCVoVAGCE3EaIP9T1A3iW0WLJ+reZlpQ==
+"@babel/helper-get-function-arity@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.8.3.tgz#b894b947bd004381ce63ea1db9f08547e920abd5"
+ integrity sha512-FVDR+Gd9iLjUMY1fzE2SR0IuaJToR4RkCDARVfsBBPSP53GEqSFjD8gNyxg246VUyc/ALRxFaAK8rVG7UT7xRA==
dependencies:
- "@babel/types" "^7.7.4"
+ "@babel/types" "^7.8.3"
-"@babel/helper-member-expression-to-functions@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.7.4.tgz#356438e2569df7321a8326644d4b790d2122cb74"
- integrity sha512-9KcA1X2E3OjXl/ykfMMInBK+uVdfIVakVe7W7Lg3wfXUNyS3Q1HWLFRwZIjhqiCGbslummPDnmb7vIekS0C1vw==
+"@babel/helper-hoist-variables@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.8.3.tgz#1dbe9b6b55d78c9b4183fc8cdc6e30ceb83b7134"
+ integrity sha512-ky1JLOjcDUtSc+xkt0xhYff7Z6ILTAHKmZLHPxAhOP0Nd77O+3nCsd6uSVYur6nJnCI029CrNbYlc0LoPfAPQg==
dependencies:
- "@babel/types" "^7.7.4"
+ "@babel/types" "^7.8.3"
-"@babel/helper-module-imports@^7.0.0", "@babel/helper-module-imports@^7.7.4":
+"@babel/helper-member-expression-to-functions@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.8.3.tgz#659b710498ea6c1d9907e0c73f206eee7dadc24c"
+ integrity sha512-fO4Egq88utkQFjbPrSHGmGLFqmrshs11d46WI+WZDESt7Wu7wN2G2Iu+NMMZJFDOVRHAMIkB5SNh30NtwCA7RA==
+ dependencies:
+ "@babel/types" "^7.8.3"
+
+"@babel/helper-module-imports@^7.0.0":
version "7.7.4"
resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.7.4.tgz#e5a92529f8888bf319a6376abfbd1cebc491ad91"
integrity sha512-dGcrX6K9l8258WFjyDLJwuVKxR4XZfU0/vTUgOQYWEnRD8mgr+p4d6fCUMq/ys0h4CCt/S5JhbvtyErjWouAUQ==
dependencies:
"@babel/types" "^7.7.4"
-"@babel/helper-module-transforms@^7.7.4", "@babel/helper-module-transforms@^7.7.5":
- version "7.7.5"
- resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.7.5.tgz#d044da7ffd91ec967db25cd6748f704b6b244835"
- integrity sha512-A7pSxyJf1gN5qXVcidwLWydjftUN878VkalhXX5iQDuGyiGK3sOrrKKHF4/A4fwHtnsotv/NipwAeLzY4KQPvw==
+"@babel/helper-module-imports@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.8.3.tgz#7fe39589b39c016331b6b8c3f441e8f0b1419498"
+ integrity sha512-R0Bx3jippsbAEtzkpZ/6FIiuzOURPcMjHp+Z6xPe6DtApDJx+w7UYyOLanZqO8+wKR9G10s/FmHXvxaMd9s6Kg==
dependencies:
- "@babel/helper-module-imports" "^7.7.4"
- "@babel/helper-simple-access" "^7.7.4"
- "@babel/helper-split-export-declaration" "^7.7.4"
- "@babel/template" "^7.7.4"
- "@babel/types" "^7.7.4"
+ "@babel/types" "^7.8.3"
+
+"@babel/helper-module-transforms@^7.9.0":
+ version "7.9.0"
+ resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.9.0.tgz#43b34dfe15961918707d247327431388e9fe96e5"
+ integrity sha512-0FvKyu0gpPfIQ8EkxlrAydOWROdHpBmiCiRwLkUiBGhCUPRRbVD2/tm3sFr/c/GWFrQ/ffutGUAnx7V0FzT2wA==
+ dependencies:
+ "@babel/helper-module-imports" "^7.8.3"
+ "@babel/helper-replace-supers" "^7.8.6"
+ "@babel/helper-simple-access" "^7.8.3"
+ "@babel/helper-split-export-declaration" "^7.8.3"
+ "@babel/template" "^7.8.6"
+ "@babel/types" "^7.9.0"
lodash "^4.17.13"
-"@babel/helper-optimise-call-expression@^7.0.0", "@babel/helper-optimise-call-expression@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.7.4.tgz#034af31370d2995242aa4df402c3b7794b2dcdf2"
- integrity sha512-VB7gWZ2fDkSuqW6b1AKXkJWO5NyNI3bFL/kK79/30moK57blr6NbH8xcl2XcKCwOmJosftWunZqfO84IGq3ZZg==
+"@babel/helper-optimise-call-expression@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.8.3.tgz#7ed071813d09c75298ef4f208956006b6111ecb9"
+ integrity sha512-Kag20n86cbO2AvHca6EJsvqAd82gc6VMGule4HwebwMlwkpXuVqrNRj6CkCV2sKxgi9MyAUnZVnZ6lJ1/vKhHQ==
dependencies:
- "@babel/types" "^7.7.4"
+ "@babel/types" "^7.8.3"
"@babel/helper-plugin-utils@^7.0.0":
version "7.0.0"
resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.0.0.tgz#bbb3fbee98661c569034237cc03967ba99b4f250"
integrity sha512-CYAOUCARwExnEixLdB6sDm2dIJ/YgEAKDM1MOeMeZu9Ld/bDgVo8aiWrXwcY7OBh+1Ea2uUcVRcxKk0GJvW7QA==
-"@babel/helper-regex@^7.0.0", "@babel/helper-regex@^7.4.4":
- version "7.5.5"
- resolved "https://registry.yarnpkg.com/@babel/helper-regex/-/helper-regex-7.5.5.tgz#0aa6824f7100a2e0e89c1527c23936c152cab351"
- integrity sha512-CkCYQLkfkiugbRDO8eZn6lRuR8kzZoGXCg3149iTk5se7g6qykSpy3+hELSwquhu+TgHn8nkLiBwHvNX8Hofcw==
+"@babel/helper-plugin-utils@^7.8.0", "@babel/helper-plugin-utils@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.8.3.tgz#9ea293be19babc0f52ff8ca88b34c3611b208670"
+ integrity sha512-j+fq49Xds2smCUNYmEHF9kGNkhbet6yVIBp4e6oeQpH1RUs/Ir06xUKzDjDkGcaaokPiTNs2JBWHjaE4csUkZQ==
+
+"@babel/helper-regex@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/helper-regex/-/helper-regex-7.8.3.tgz#139772607d51b93f23effe72105b319d2a4c6965"
+ integrity sha512-BWt0QtYv/cg/NecOAZMdcn/waj/5P26DR4mVLXfFtDokSR6fyuG0Pj+e2FqtSME+MqED1khnSMulkmGl8qWiUQ==
dependencies:
lodash "^4.17.13"
-"@babel/helper-remap-async-to-generator@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.7.4.tgz#c68c2407350d9af0e061ed6726afb4fff16d0234"
- integrity sha512-Sk4xmtVdM9sA/jCI80f+KS+Md+ZHIpjuqmYPk1M7F/upHou5e4ReYmExAiu6PVe65BhJPZA2CY9x9k4BqE5klw==
+"@babel/helper-remap-async-to-generator@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.8.3.tgz#273c600d8b9bf5006142c1e35887d555c12edd86"
+ integrity sha512-kgwDmw4fCg7AVgS4DukQR/roGp+jP+XluJE5hsRZwxCYGg+Rv9wSGErDWhlI90FODdYfd4xG4AQRiMDjjN0GzA==
dependencies:
- "@babel/helper-annotate-as-pure" "^7.7.4"
- "@babel/helper-wrap-function" "^7.7.4"
- "@babel/template" "^7.7.4"
- "@babel/traverse" "^7.7.4"
- "@babel/types" "^7.7.4"
+ "@babel/helper-annotate-as-pure" "^7.8.3"
+ "@babel/helper-wrap-function" "^7.8.3"
+ "@babel/template" "^7.8.3"
+ "@babel/traverse" "^7.8.3"
+ "@babel/types" "^7.8.3"
-"@babel/helper-replace-supers@^7.4.0", "@babel/helper-replace-supers@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/helper-replace-supers/-/helper-replace-supers-7.7.4.tgz#3c881a6a6a7571275a72d82e6107126ec9e2cdd2"
- integrity sha512-pP0tfgg9hsZWo5ZboYGuBn/bbYT/hdLPVSS4NMmiRJdwWhP0IznPwN9AE1JwyGsjSPLC364I0Qh5p+EPkGPNpg==
+"@babel/helper-replace-supers@^7.8.3", "@babel/helper-replace-supers@^7.8.6", "@babel/helper-replace-supers@^7.9.6":
+ version "7.9.6"
+ resolved "https://registry.yarnpkg.com/@babel/helper-replace-supers/-/helper-replace-supers-7.9.6.tgz#03149d7e6a5586ab6764996cd31d6981a17e1444"
+ integrity sha512-qX+chbxkbArLyCImk3bWV+jB5gTNU/rsze+JlcF6Nf8tVTigPJSI1o1oBow/9Resa1yehUO9lIipsmu9oG4RzA==
dependencies:
- "@babel/helper-member-expression-to-functions" "^7.7.4"
- "@babel/helper-optimise-call-expression" "^7.7.4"
- "@babel/traverse" "^7.7.4"
- "@babel/types" "^7.7.4"
+ "@babel/helper-member-expression-to-functions" "^7.8.3"
+ "@babel/helper-optimise-call-expression" "^7.8.3"
+ "@babel/traverse" "^7.9.6"
+ "@babel/types" "^7.9.6"
-"@babel/helper-simple-access@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/helper-simple-access/-/helper-simple-access-7.7.4.tgz#a169a0adb1b5f418cfc19f22586b2ebf58a9a294"
- integrity sha512-zK7THeEXfan7UlWsG2A6CI/L9jVnI5+xxKZOdej39Y0YtDYKx9raHk5F2EtK9K8DHRTihYwg20ADt9S36GR78A==
+"@babel/helper-simple-access@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/helper-simple-access/-/helper-simple-access-7.8.3.tgz#7f8109928b4dab4654076986af575231deb639ae"
+ integrity sha512-VNGUDjx5cCWg4vvCTR8qQ7YJYZ+HBjxOgXEl7ounz+4Sn7+LMD3CFrCTEU6/qXKbA2nKg21CwhhBzO0RpRbdCw==
dependencies:
- "@babel/template" "^7.7.4"
- "@babel/types" "^7.7.4"
+ "@babel/template" "^7.8.3"
+ "@babel/types" "^7.8.3"
-"@babel/helper-split-export-declaration@^7.4.0", "@babel/helper-split-export-declaration@^7.7.4":
+"@babel/helper-split-export-declaration@^7.7.4":
version "7.7.4"
resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.7.4.tgz#57292af60443c4a3622cf74040ddc28e68336fd8"
integrity sha512-guAg1SXFcVr04Guk9eq0S4/rWS++sbmyqosJzVs8+1fH5NI+ZcmkaSkc7dmtAFbHFva6yRJnjW3yAcGxjueDug==
dependencies:
"@babel/types" "^7.7.4"
-"@babel/helper-wrap-function@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/helper-wrap-function/-/helper-wrap-function-7.7.4.tgz#37ab7fed5150e22d9d7266e830072c0cdd8baace"
- integrity sha512-VsfzZt6wmsocOaVU0OokwrIytHND55yvyT4BPB9AIIgwr8+x7617hetdJTsuGwygN5RC6mxA9EJztTjuwm2ofg==
+"@babel/helper-split-export-declaration@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.8.3.tgz#31a9f30070f91368a7182cf05f831781065fc7a9"
+ integrity sha512-3x3yOeyBhW851hroze7ElzdkeRXQYQbFIb7gLK1WQYsw2GWDay5gAJNw1sWJ0VFP6z5J1whqeXH/WCdCjZv6dA==
dependencies:
- "@babel/helper-function-name" "^7.7.4"
- "@babel/template" "^7.7.4"
- "@babel/traverse" "^7.7.4"
- "@babel/types" "^7.7.4"
+ "@babel/types" "^7.8.3"
-"@babel/helpers@^7.4.3", "@babel/helpers@^7.7.4":
+"@babel/helper-validator-identifier@^7.9.0", "@babel/helper-validator-identifier@^7.9.5":
+ version "7.9.5"
+ resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.9.5.tgz#90977a8e6fbf6b431a7dc31752eee233bf052d80"
+ integrity sha512-/8arLKUFq882w4tWGj9JYzRpAlZgiWUJ+dtteNTDqrRBz9Iguck9Rn3ykuBDoUwh2TO4tSAJlrxDUOXWklJe4g==
+
+"@babel/helper-wrap-function@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/helper-wrap-function/-/helper-wrap-function-7.8.3.tgz#9dbdb2bb55ef14aaa01fe8c99b629bd5352d8610"
+ integrity sha512-LACJrbUET9cQDzb6kG7EeD7+7doC3JNvUgTEQOx2qaO1fKlzE/Bf05qs9w1oXQMmXlPO65lC3Tq9S6gZpTErEQ==
+ dependencies:
+ "@babel/helper-function-name" "^7.8.3"
+ "@babel/template" "^7.8.3"
+ "@babel/traverse" "^7.8.3"
+ "@babel/types" "^7.8.3"
+
+"@babel/helpers@^7.7.4":
version "7.7.4"
resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.7.4.tgz#62c215b9e6c712dadc15a9a0dcab76c92a940302"
integrity sha512-ak5NGZGJ6LV85Q1Zc9gn2n+ayXOizryhjSUBTdu5ih1tlVCJeuQENzc4ItyCVhINVXvIT/ZQ4mheGIsfBkpskg==
@@ -279,6 +375,15 @@
"@babel/traverse" "^7.7.4"
"@babel/types" "^7.7.4"
+"@babel/helpers@^7.9.0":
+ version "7.9.6"
+ resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.9.6.tgz#092c774743471d0bb6c7de3ad465ab3d3486d580"
+ integrity sha512-tI4bUbldloLcHWoRUMAj4g1bF313M/o6fBKhIsb3QnGVPwRm9JsNf/gqMkQ7zjqReABiffPV6RWj7hEglID5Iw==
+ dependencies:
+ "@babel/template" "^7.8.3"
+ "@babel/traverse" "^7.9.6"
+ "@babel/types" "^7.9.6"
+
"@babel/highlight@^7.0.0":
version "7.5.0"
resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.5.0.tgz#56d11312bd9248fa619591d02472be6e8cb32540"
@@ -288,126 +393,150 @@
esutils "^2.0.2"
js-tokens "^4.0.0"
-"@babel/parser@^7.0.0", "@babel/parser@^7.1.0", "@babel/parser@^7.4.3", "@babel/parser@^7.7.4", "@babel/parser@^7.7.7":
+"@babel/highlight@^7.8.3":
+ version "7.9.0"
+ resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.9.0.tgz#4e9b45ccb82b79607271b2979ad82c7b68163079"
+ integrity sha512-lJZPilxX7Op3Nv/2cvFdnlepPXDxi29wxteT57Q965oc5R9v86ztx0jfxVrTcBk8C2kcPkkDa2Z4T3ZsPPVWsQ==
+ dependencies:
+ "@babel/helper-validator-identifier" "^7.9.0"
+ chalk "^2.0.0"
+ js-tokens "^4.0.0"
+
+"@babel/parser@^7.1.0", "@babel/parser@^7.4.3", "@babel/parser@^7.7.4", "@babel/parser@^7.7.7":
version "7.7.7"
resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.7.7.tgz#1b886595419cf92d811316d5b715a53ff38b4937"
integrity sha512-WtTZMZAZLbeymhkd/sEaPD8IQyGAhmuTuvTzLiCFM7iXiVdY0gc0IaI+cW0fh1BnSMbJSzXX6/fHllgHKwHhXw==
-"@babel/plugin-proposal-async-generator-functions@^7.2.0", "@babel/plugin-proposal-async-generator-functions@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.7.4.tgz#0351c5ac0a9e927845fffd5b82af476947b7ce6d"
- integrity sha512-1ypyZvGRXriY/QP668+s8sFr2mqinhkRDMPSQLNghCQE+GAkFtp+wkHVvg2+Hdki8gwP+NFzJBJ/N1BfzCCDEw==
- dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
- "@babel/helper-remap-async-to-generator" "^7.7.4"
- "@babel/plugin-syntax-async-generators" "^7.7.4"
+"@babel/parser@^7.7.0", "@babel/parser@^7.8.6", "@babel/parser@^7.9.0", "@babel/parser@^7.9.6":
+ version "7.9.6"
+ resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.9.6.tgz#3b1bbb30dabe600cd72db58720998376ff653bc7"
+ integrity sha512-AoeIEJn8vt+d/6+PXDRPaksYhnlbMIiejioBZvvMQsOjW/JYK6k/0dKnvvP3EhK5GfMBWDPtrxRtegWdAcdq9Q==
-"@babel/plugin-proposal-class-properties@7.4.0":
- version "7.4.0"
- resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.4.0.tgz#d70db61a2f1fd79de927eea91f6411c964e084b8"
- integrity sha512-t2ECPNOXsIeK1JxJNKmgbzQtoG27KIlVE61vTqX0DKR9E9sZlVVxWUtEW9D5FlZ8b8j7SBNCHY47GgPKCKlpPg==
+"@babel/plugin-proposal-async-generator-functions@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.8.3.tgz#bad329c670b382589721b27540c7d288601c6e6f"
+ integrity sha512-NZ9zLv848JsV3hs8ryEh7Uaz/0KsmPLqv0+PdkDJL1cJy0K4kOCFa8zc1E3mp+RHPQcpdfb/6GovEsW4VDrOMw==
dependencies:
- "@babel/helper-create-class-features-plugin" "^7.4.0"
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
+ "@babel/helper-remap-async-to-generator" "^7.8.3"
+ "@babel/plugin-syntax-async-generators" "^7.8.0"
-"@babel/plugin-proposal-decorators@7.4.0":
- version "7.4.0"
- resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-decorators/-/plugin-proposal-decorators-7.4.0.tgz#8e1bfd83efa54a5f662033afcc2b8e701f4bb3a9"
- integrity sha512-d08TLmXeK/XbgCo7ZeZ+JaeZDtDai/2ctapTRsWWkkmy7G/cqz8DQN/HlWG7RR4YmfXxmExsbU3SuCjlM7AtUg==
+"@babel/plugin-proposal-class-properties@7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.8.3.tgz#5e06654af5cd04b608915aada9b2a6788004464e"
+ integrity sha512-EqFhbo7IosdgPgZggHaNObkmO1kNUe3slaKu54d5OWvy+p9QIKOzK1GAEpAIsZtWVtPXUHSMcT4smvDrCfY4AA==
dependencies:
- "@babel/helper-create-class-features-plugin" "^7.4.0"
- "@babel/helper-plugin-utils" "^7.0.0"
- "@babel/plugin-syntax-decorators" "^7.2.0"
+ "@babel/helper-create-class-features-plugin" "^7.8.3"
+ "@babel/helper-plugin-utils" "^7.8.3"
-"@babel/plugin-proposal-dynamic-import@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-dynamic-import/-/plugin-proposal-dynamic-import-7.7.4.tgz#dde64a7f127691758cbfed6cf70de0fa5879d52d"
- integrity sha512-StH+nGAdO6qDB1l8sZ5UBV8AC3F2VW2I8Vfld73TMKyptMU9DY5YsJAS8U81+vEtxcH3Y/La0wG0btDrhpnhjQ==
+"@babel/plugin-proposal-decorators@7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-decorators/-/plugin-proposal-decorators-7.8.3.tgz#2156860ab65c5abf068c3f67042184041066543e"
+ integrity sha512-e3RvdvS4qPJVTe288DlXjwKflpfy1hr0j5dz5WpIYYeP7vQZg2WfAEIp8k5/Lwis/m5REXEteIz6rrcDtXXG7w==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
- "@babel/plugin-syntax-dynamic-import" "^7.7.4"
+ "@babel/helper-create-class-features-plugin" "^7.8.3"
+ "@babel/helper-plugin-utils" "^7.8.3"
+ "@babel/plugin-syntax-decorators" "^7.8.3"
-"@babel/plugin-proposal-json-strings@^7.2.0", "@babel/plugin-proposal-json-strings@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-json-strings/-/plugin-proposal-json-strings-7.7.4.tgz#7700a6bfda771d8dc81973249eac416c6b4c697d"
- integrity sha512-wQvt3akcBTfLU/wYoqm/ws7YOAQKu8EVJEvHip/mzkNtjaclQoCCIqKXFP5/eyfnfbQCDV3OLRIK3mIVyXuZlw==
+"@babel/plugin-proposal-dynamic-import@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-dynamic-import/-/plugin-proposal-dynamic-import-7.8.3.tgz#38c4fe555744826e97e2ae930b0fb4cc07e66054"
+ integrity sha512-NyaBbyLFXFLT9FP+zk0kYlUlA8XtCUbehs67F0nnEg7KICgMc2mNkIeu9TYhKzyXMkrapZFwAhXLdnt4IYHy1w==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
- "@babel/plugin-syntax-json-strings" "^7.7.4"
+ "@babel/helper-plugin-utils" "^7.8.3"
+ "@babel/plugin-syntax-dynamic-import" "^7.8.0"
-"@babel/plugin-proposal-object-rest-spread@7.4.3":
- version "7.4.3"
- resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.4.3.tgz#be27cd416eceeba84141305b93c282f5de23bbb4"
- integrity sha512-xC//6DNSSHVjq8O2ge0dyYlhshsH4T7XdCVoxbi5HzLYWfsC5ooFlJjrXk8RcAT+hjHAK9UjBXdylzSoDK3t4g==
+"@babel/plugin-proposal-json-strings@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-json-strings/-/plugin-proposal-json-strings-7.8.3.tgz#da5216b238a98b58a1e05d6852104b10f9a70d6b"
+ integrity sha512-KGhQNZ3TVCQG/MjRbAUwuH+14y9q0tpxs1nWWs3pbSleRdDro9SAMMDyye8HhY1gqZ7/NqIc8SKhya0wRDgP1Q==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
- "@babel/plugin-syntax-object-rest-spread" "^7.2.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
+ "@babel/plugin-syntax-json-strings" "^7.8.0"
-"@babel/plugin-proposal-object-rest-spread@^7.4.3", "@babel/plugin-proposal-object-rest-spread@^7.7.7":
- version "7.7.7"
- resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.7.7.tgz#9f27075004ab99be08c5c1bd653a2985813cb370"
- integrity sha512-3qp9I8lelgzNedI3hrhkvhaEYree6+WHnyA/q4Dza9z7iEIs1eyhWyJnetk3jJ69RT0AT4G0UhEGwyGFJ7GUuQ==
+"@babel/plugin-proposal-nullish-coalescing-operator@7.8.3", "@babel/plugin-proposal-nullish-coalescing-operator@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-nullish-coalescing-operator/-/plugin-proposal-nullish-coalescing-operator-7.8.3.tgz#e4572253fdeed65cddeecfdab3f928afeb2fd5d2"
+ integrity sha512-TS9MlfzXpXKt6YYomudb/KU7nQI6/xnapG6in1uZxoxDghuSMZsPb6D2fyUwNYSAp4l1iR7QtFOjkqcRYcUsfw==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
- "@babel/plugin-syntax-object-rest-spread" "^7.7.4"
+ "@babel/helper-plugin-utils" "^7.8.3"
+ "@babel/plugin-syntax-nullish-coalescing-operator" "^7.8.0"
-"@babel/plugin-proposal-optional-catch-binding@^7.2.0", "@babel/plugin-proposal-optional-catch-binding@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-optional-catch-binding/-/plugin-proposal-optional-catch-binding-7.7.4.tgz#ec21e8aeb09ec6711bc0a39ca49520abee1de379"
- integrity sha512-DyM7U2bnsQerCQ+sejcTNZh8KQEUuC3ufzdnVnSiUv/qoGJp2Z3hanKL18KDhsBT5Wj6a7CMT5mdyCNJsEaA9w==
+"@babel/plugin-proposal-numeric-separator@7.8.3", "@babel/plugin-proposal-numeric-separator@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-numeric-separator/-/plugin-proposal-numeric-separator-7.8.3.tgz#5d6769409699ec9b3b68684cd8116cedff93bad8"
+ integrity sha512-jWioO1s6R/R+wEHizfaScNsAx+xKgwTLNXSh7tTC4Usj3ItsPEhYkEpU4h+lpnBwq7NBVOJXfO6cRFYcX69JUQ==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
- "@babel/plugin-syntax-optional-catch-binding" "^7.7.4"
+ "@babel/helper-plugin-utils" "^7.8.3"
+ "@babel/plugin-syntax-numeric-separator" "^7.8.3"
-"@babel/plugin-proposal-unicode-property-regex@^7.4.0", "@babel/plugin-proposal-unicode-property-regex@^7.7.7":
- version "7.7.7"
- resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-unicode-property-regex/-/plugin-proposal-unicode-property-regex-7.7.7.tgz#433fa9dac64f953c12578b29633f456b68831c4e"
- integrity sha512-80PbkKyORBUVm1fbTLrHpYdJxMThzM1UqFGh0ALEhO9TYbG86Ah9zQYAB/84axz2vcxefDLdZwWwZNlYARlu9w==
+"@babel/plugin-proposal-object-rest-spread@^7.9.0", "@babel/plugin-proposal-object-rest-spread@^7.9.6":
+ version "7.9.6"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.9.6.tgz#7a093586fcb18b08266eb1a7177da671ac575b63"
+ integrity sha512-Ga6/fhGqA9Hj+y6whNpPv8psyaK5xzrQwSPsGPloVkvmH+PqW1ixdnfJ9uIO06OjQNYol3PMnfmJ8vfZtkzF+A==
dependencies:
- "@babel/helper-create-regexp-features-plugin" "^7.7.4"
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
+ "@babel/plugin-syntax-object-rest-spread" "^7.8.0"
+ "@babel/plugin-transform-parameters" "^7.9.5"
-"@babel/plugin-syntax-async-generators@^7.2.0", "@babel/plugin-syntax-async-generators@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.7.4.tgz#331aaf310a10c80c44a66b238b6e49132bd3c889"
- integrity sha512-Li4+EjSpBgxcsmeEF8IFcfV/+yJGxHXDirDkEoyFjumuwbmfCVHUt0HuowD/iGM7OhIRyXJH9YXxqiH6N815+g==
+"@babel/plugin-proposal-optional-catch-binding@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-optional-catch-binding/-/plugin-proposal-optional-catch-binding-7.8.3.tgz#9dee96ab1650eed88646ae9734ca167ac4a9c5c9"
+ integrity sha512-0gkX7J7E+AtAw9fcwlVQj8peP61qhdg/89D5swOkjYbkboA2CVckn3kiyum1DE0wskGb7KJJxBdyEBApDLLVdw==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
+ "@babel/plugin-syntax-optional-catch-binding" "^7.8.0"
-"@babel/plugin-syntax-decorators@^7.2.0":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-decorators/-/plugin-syntax-decorators-7.7.4.tgz#3c91cfee2a111663ff3ac21b851140f5a52a4e0b"
- integrity sha512-0oNLWNH4k5ZbBVfAwiTU53rKFWIeTh6ZlaWOXWJc4ywxs0tjz5fc3uZ6jKAnZSxN98eXVgg7bJIuzjX+3SXY+A==
+"@babel/plugin-proposal-optional-chaining@7.9.0", "@babel/plugin-proposal-optional-chaining@^7.9.0":
+ version "7.9.0"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.9.0.tgz#31db16b154c39d6b8a645292472b98394c292a58"
+ integrity sha512-NDn5tu3tcv4W30jNhmc2hyD5c56G6cXx4TesJubhxrJeCvuuMpttxr0OnNCqbZGhFjLrg+NIhxxC+BK5F6yS3w==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
+ "@babel/plugin-syntax-optional-chaining" "^7.8.0"
-"@babel/plugin-syntax-dynamic-import@7.2.0":
- version "7.2.0"
- resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-dynamic-import/-/plugin-syntax-dynamic-import-7.2.0.tgz#69c159ffaf4998122161ad8ebc5e6d1f55df8612"
- integrity sha512-mVxuJ0YroI/h/tbFTPGZR8cv6ai+STMKNBq0f8hFxsxWjl94qqhsb+wXbpNMDPU3cfR1TIsVFzU3nXyZMqyK4w==
+"@babel/plugin-proposal-unicode-property-regex@^7.4.4", "@babel/plugin-proposal-unicode-property-regex@^7.8.3":
+ version "7.8.8"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-unicode-property-regex/-/plugin-proposal-unicode-property-regex-7.8.8.tgz#ee3a95e90cdc04fe8cd92ec3279fa017d68a0d1d"
+ integrity sha512-EVhjVsMpbhLw9ZfHWSx2iy13Q8Z/eg8e8ccVWt23sWQK5l1UdkoLJPN5w69UA4uITGBnEZD2JOe4QOHycYKv8A==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-create-regexp-features-plugin" "^7.8.8"
+ "@babel/helper-plugin-utils" "^7.8.3"
-"@babel/plugin-syntax-dynamic-import@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-dynamic-import/-/plugin-syntax-dynamic-import-7.7.4.tgz#29ca3b4415abfe4a5ec381e903862ad1a54c3aec"
- integrity sha512-jHQW0vbRGvwQNgyVxwDh4yuXu4bH1f5/EICJLAhl1SblLs2CDhrsmCk+v5XLdE9wxtAFRyxx+P//Iw+a5L/tTg==
+"@babel/plugin-syntax-async-generators@^7.8.0":
+ version "7.8.4"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz#a983fb1aeb2ec3f6ed042a210f640e90e786fe0d"
+ integrity sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-plugin-utils" "^7.8.0"
-"@babel/plugin-syntax-flow@^7.2.0":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-flow/-/plugin-syntax-flow-7.7.4.tgz#6d91b59e1a0e4c17f36af2e10dd64ef220919d7b"
- integrity sha512-2AMAWl5PsmM5KPkB22cvOkUyWk6MjUaqhHNU5nSPUl/ns3j5qLfw2SuYP5RbVZ0tfLvePr4zUScbICtDP2CUNw==
+"@babel/plugin-syntax-decorators@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-decorators/-/plugin-syntax-decorators-7.8.3.tgz#8d2c15a9f1af624b0025f961682a9d53d3001bda"
+ integrity sha512-8Hg4dNNT9/LcA1zQlfwuKR8BUc/if7Q7NkTam9sGTcJphLwpf2g4S42uhspQrIrR+dpzE0dtTqBVFoHl8GtnnQ==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
-"@babel/plugin-syntax-json-strings@^7.2.0", "@babel/plugin-syntax-json-strings@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.7.4.tgz#86e63f7d2e22f9e27129ac4e83ea989a382e86cc"
- integrity sha512-QpGupahTQW1mHRXddMG5srgpHWqRLwJnJZKXTigB9RPFCCGbDGCgBeM/iC82ICXp414WeYx/tD54w7M2qRqTMg==
+"@babel/plugin-syntax-dynamic-import@^7.8.0":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-dynamic-import/-/plugin-syntax-dynamic-import-7.8.3.tgz#62bf98b2da3cd21d626154fc96ee5b3cb68eacb3"
+ integrity sha512-5gdGbFon+PszYzqs83S3E5mpi7/y/8M9eC90MRTZfduQOYW76ig6SOSPNe41IG5LoP3FGBn2N0RjVDSQiS94kQ==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-plugin-utils" "^7.8.0"
+
+"@babel/plugin-syntax-flow@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-flow/-/plugin-syntax-flow-7.8.3.tgz#f2c883bd61a6316f2c89380ae5122f923ba4527f"
+ integrity sha512-innAx3bUbA0KSYj2E2MNFSn9hiCeowOFLxlsuhXzw8hMQnzkDomUr9QCD7E9VF60NmnG1sNTuuv6Qf4f8INYsg==
+ dependencies:
+ "@babel/helper-plugin-utils" "^7.8.3"
+
+"@babel/plugin-syntax-json-strings@^7.8.0":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz#01ca21b668cd8218c9e640cb6dd88c5412b2c96a"
+ integrity sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==
+ dependencies:
+ "@babel/helper-plugin-utils" "^7.8.0"
"@babel/plugin-syntax-jsx@^7.7.4":
version "7.7.4"
@@ -416,255 +545,261 @@
dependencies:
"@babel/helper-plugin-utils" "^7.0.0"
-"@babel/plugin-syntax-object-rest-spread@^7.0.0", "@babel/plugin-syntax-object-rest-spread@^7.2.0", "@babel/plugin-syntax-object-rest-spread@^7.7.4":
+"@babel/plugin-syntax-jsx@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.8.3.tgz#521b06c83c40480f1e58b4fd33b92eceb1d6ea94"
+ integrity sha512-WxdW9xyLgBdefoo0Ynn3MRSkhe5tFVxxKNVdnZSh318WrG2e2jH+E9wd/++JsqcLJZPfz87njQJ8j2Upjm0M0A==
+ dependencies:
+ "@babel/helper-plugin-utils" "^7.8.3"
+
+"@babel/plugin-syntax-nullish-coalescing-operator@^7.8.0":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz#167ed70368886081f74b5c36c65a88c03b66d1a9"
+ integrity sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==
+ dependencies:
+ "@babel/helper-plugin-utils" "^7.8.0"
+
+"@babel/plugin-syntax-numeric-separator@^7.8.0", "@babel/plugin-syntax-numeric-separator@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.8.3.tgz#0e3fb63e09bea1b11e96467271c8308007e7c41f"
+ integrity sha512-H7dCMAdN83PcCmqmkHB5dtp+Xa9a6LKSvA2hiFBC/5alSHxM5VgWZXFqDi0YFe8XNGT6iCa+z4V4zSt/PdZ7Dw==
+ dependencies:
+ "@babel/helper-plugin-utils" "^7.8.3"
+
+"@babel/plugin-syntax-object-rest-spread@^7.0.0":
version "7.7.4"
resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.7.4.tgz#47cf220d19d6d0d7b154304701f468fc1cc6ff46"
integrity sha512-mObR+r+KZq0XhRVS2BrBKBpr5jqrqzlPvS9C9vuOf5ilSwzloAl7RPWLrgKdWS6IreaVrjHxTjtyqFiOisaCwg==
dependencies:
"@babel/helper-plugin-utils" "^7.0.0"
-"@babel/plugin-syntax-optional-catch-binding@^7.2.0", "@babel/plugin-syntax-optional-catch-binding@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.7.4.tgz#a3e38f59f4b6233867b4a92dcb0ee05b2c334aa6"
- integrity sha512-4ZSuzWgFxqHRE31Glu+fEr/MirNZOMYmD/0BhBWyLyOOQz/gTAl7QmWm2hX1QxEIXsr2vkdlwxIzTyiYRC4xcQ==
+"@babel/plugin-syntax-object-rest-spread@^7.8.0":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz#60e225edcbd98a640332a2e72dd3e66f1af55871"
+ integrity sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-plugin-utils" "^7.8.0"
-"@babel/plugin-syntax-top-level-await@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.7.4.tgz#bd7d8fa7b9fee793a36e4027fd6dd1aa32f946da"
- integrity sha512-wdsOw0MvkL1UIgiQ/IFr3ETcfv1xb8RMM0H9wbiDyLaJFyiDg5oZvDLCXosIXmFeIlweML5iOBXAkqddkYNizg==
+"@babel/plugin-syntax-optional-catch-binding@^7.8.0":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz#6111a265bcfb020eb9efd0fdfd7d26402b9ed6c1"
+ integrity sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-plugin-utils" "^7.8.0"
-"@babel/plugin-syntax-typescript@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.7.4.tgz#5d037ffa10f3b25a16f32570ebbe7a8c2efa304b"
- integrity sha512-77blgY18Hud4NM1ggTA8xVT/dBENQf17OpiToSa2jSmEY3fWXD2jwrdVlO4kq5yzUTeF15WSQ6b4fByNvJcjpQ==
+"@babel/plugin-syntax-optional-chaining@^7.8.0":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz#4f69c2ab95167e0180cd5336613f8c5788f7d48a"
+ integrity sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-plugin-utils" "^7.8.0"
-"@babel/plugin-transform-arrow-functions@^7.2.0", "@babel/plugin-transform-arrow-functions@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.7.4.tgz#76309bd578addd8aee3b379d809c802305a98a12"
- integrity sha512-zUXy3e8jBNPiffmqkHRNDdZM2r8DWhCB7HhcoyZjiK1TxYEluLHAvQuYnTT+ARqRpabWqy/NHkO6e3MsYB5YfA==
+"@babel/plugin-syntax-top-level-await@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.8.3.tgz#3acdece695e6b13aaf57fc291d1a800950c71391"
+ integrity sha512-kwj1j9lL/6Wd0hROD3b/OZZ7MSrZLqqn9RAZ5+cYYsflQ9HZBIKCUkr3+uL1MEJ1NePiUbf98jjiMQSv0NMR9g==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
-"@babel/plugin-transform-async-to-generator@^7.4.0", "@babel/plugin-transform-async-to-generator@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-async-to-generator/-/plugin-transform-async-to-generator-7.7.4.tgz#694cbeae6d613a34ef0292713fa42fb45c4470ba"
- integrity sha512-zpUTZphp5nHokuy8yLlyafxCJ0rSlFoSHypTUWgpdwoDXWQcseaect7cJ8Ppk6nunOM6+5rPMkod4OYKPR5MUg==
+"@babel/plugin-syntax-typescript@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.8.3.tgz#c1f659dda97711a569cef75275f7e15dcaa6cabc"
+ integrity sha512-GO1MQ/SGGGoiEXY0e0bSpHimJvxqB7lktLLIq2pv8xG7WZ8IMEle74jIe1FhprHBWjwjZtXHkycDLZXIWM5Wfg==
dependencies:
- "@babel/helper-module-imports" "^7.7.4"
- "@babel/helper-plugin-utils" "^7.0.0"
- "@babel/helper-remap-async-to-generator" "^7.7.4"
+ "@babel/helper-plugin-utils" "^7.8.3"
-"@babel/plugin-transform-block-scoped-functions@^7.2.0", "@babel/plugin-transform-block-scoped-functions@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.7.4.tgz#d0d9d5c269c78eaea76227ace214b8d01e4d837b"
- integrity sha512-kqtQzwtKcpPclHYjLK//3lH8OFsCDuDJBaFhVwf8kqdnF6MN4l618UDlcA7TfRs3FayrHj+svYnSX8MC9zmUyQ==
+"@babel/plugin-transform-arrow-functions@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.8.3.tgz#82776c2ed0cd9e1a49956daeb896024c9473b8b6"
+ integrity sha512-0MRF+KC8EqH4dbuITCWwPSzsyO3HIWWlm30v8BbbpOrS1B++isGxPnnuq/IZvOX5J2D/p7DQalQm+/2PnlKGxg==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
-"@babel/plugin-transform-block-scoping@^7.4.0", "@babel/plugin-transform-block-scoping@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.7.4.tgz#200aad0dcd6bb80372f94d9e628ea062c58bf224"
- integrity sha512-2VBe9u0G+fDt9B5OV5DQH4KBf5DoiNkwFKOz0TCvBWvdAN2rOykCTkrL+jTLxfCAm76l9Qo5OqL7HBOx2dWggg==
+"@babel/plugin-transform-async-to-generator@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-async-to-generator/-/plugin-transform-async-to-generator-7.8.3.tgz#4308fad0d9409d71eafb9b1a6ee35f9d64b64086"
+ integrity sha512-imt9tFLD9ogt56Dd5CI/6XgpukMwd/fLGSrix2httihVe7LOGVPhyhMh1BU5kDM7iHD08i8uUtmV2sWaBFlHVQ==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-module-imports" "^7.8.3"
+ "@babel/helper-plugin-utils" "^7.8.3"
+ "@babel/helper-remap-async-to-generator" "^7.8.3"
+
+"@babel/plugin-transform-block-scoped-functions@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.8.3.tgz#437eec5b799b5852072084b3ae5ef66e8349e8a3"
+ integrity sha512-vo4F2OewqjbB1+yaJ7k2EJFHlTP3jR634Z9Cj9itpqNjuLXvhlVxgnjsHsdRgASR8xYDrx6onw4vW5H6We0Jmg==
+ dependencies:
+ "@babel/helper-plugin-utils" "^7.8.3"
+
+"@babel/plugin-transform-block-scoping@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.8.3.tgz#97d35dab66857a437c166358b91d09050c868f3a"
+ integrity sha512-pGnYfm7RNRgYRi7bids5bHluENHqJhrV4bCZRwc5GamaWIIs07N4rZECcmJL6ZClwjDz1GbdMZFtPs27hTB06w==
+ dependencies:
+ "@babel/helper-plugin-utils" "^7.8.3"
lodash "^4.17.13"
-"@babel/plugin-transform-classes@7.4.3":
- version "7.4.3"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-classes/-/plugin-transform-classes-7.4.3.tgz#adc7a1137ab4287a555d429cc56ecde8f40c062c"
- integrity sha512-PUaIKyFUDtG6jF5DUJOfkBdwAS/kFFV3XFk7Nn0a6vR7ZT8jYw5cGtIlat77wcnd0C6ViGqo/wyNf4ZHytF/nQ==
+"@babel/plugin-transform-classes@^7.9.0", "@babel/plugin-transform-classes@^7.9.5":
+ version "7.9.5"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-classes/-/plugin-transform-classes-7.9.5.tgz#800597ddb8aefc2c293ed27459c1fcc935a26c2c"
+ integrity sha512-x2kZoIuLC//O5iA7PEvecB105o7TLzZo8ofBVhP79N+DO3jaX+KYfww9TQcfBEZD0nikNyYcGB1IKtRq36rdmg==
dependencies:
- "@babel/helper-annotate-as-pure" "^7.0.0"
- "@babel/helper-define-map" "^7.4.0"
- "@babel/helper-function-name" "^7.1.0"
- "@babel/helper-optimise-call-expression" "^7.0.0"
- "@babel/helper-plugin-utils" "^7.0.0"
- "@babel/helper-replace-supers" "^7.4.0"
- "@babel/helper-split-export-declaration" "^7.4.0"
+ "@babel/helper-annotate-as-pure" "^7.8.3"
+ "@babel/helper-define-map" "^7.8.3"
+ "@babel/helper-function-name" "^7.9.5"
+ "@babel/helper-optimise-call-expression" "^7.8.3"
+ "@babel/helper-plugin-utils" "^7.8.3"
+ "@babel/helper-replace-supers" "^7.8.6"
+ "@babel/helper-split-export-declaration" "^7.8.3"
globals "^11.1.0"
-"@babel/plugin-transform-classes@^7.4.3", "@babel/plugin-transform-classes@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-classes/-/plugin-transform-classes-7.7.4.tgz#c92c14be0a1399e15df72667067a8f510c9400ec"
- integrity sha512-sK1mjWat7K+buWRuImEzjNf68qrKcrddtpQo3swi9j7dUcG6y6R6+Di039QN2bD1dykeswlagupEmpOatFHHUg==
+"@babel/plugin-transform-computed-properties@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.8.3.tgz#96d0d28b7f7ce4eb5b120bb2e0e943343c86f81b"
+ integrity sha512-O5hiIpSyOGdrQZRQ2ccwtTVkgUDBBiCuK//4RJ6UfePllUTCENOzKxfh6ulckXKc0DixTFLCfb2HVkNA7aDpzA==
dependencies:
- "@babel/helper-annotate-as-pure" "^7.7.4"
- "@babel/helper-define-map" "^7.7.4"
- "@babel/helper-function-name" "^7.7.4"
- "@babel/helper-optimise-call-expression" "^7.7.4"
- "@babel/helper-plugin-utils" "^7.0.0"
- "@babel/helper-replace-supers" "^7.7.4"
- "@babel/helper-split-export-declaration" "^7.7.4"
- globals "^11.1.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
-"@babel/plugin-transform-computed-properties@^7.2.0", "@babel/plugin-transform-computed-properties@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.7.4.tgz#e856c1628d3238ffe12d668eb42559f79a81910d"
- integrity sha512-bSNsOsZnlpLLyQew35rl4Fma3yKWqK3ImWMSC/Nc+6nGjC9s5NFWAer1YQ899/6s9HxO2zQC1WoFNfkOqRkqRQ==
+"@babel/plugin-transform-destructuring@^7.8.3", "@babel/plugin-transform-destructuring@^7.9.5":
+ version "7.9.5"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.9.5.tgz#72c97cf5f38604aea3abf3b935b0e17b1db76a50"
+ integrity sha512-j3OEsGel8nHL/iusv/mRd5fYZ3DrOxWC82x0ogmdN/vHfAP4MYw+AFKYanzWlktNwikKvlzUV//afBW5FTp17Q==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
-"@babel/plugin-transform-destructuring@7.4.3":
- version "7.4.3"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.4.3.tgz#1a95f5ca2bf2f91ef0648d5de38a8d472da4350f"
- integrity sha512-rVTLLZpydDFDyN4qnXdzwoVpk1oaXHIvPEOkOLyr88o7oHxVc/LyrnDx+amuBWGOwUb7D1s/uLsKBNTx08htZg==
+"@babel/plugin-transform-dotall-regex@^7.4.4", "@babel/plugin-transform-dotall-regex@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.8.3.tgz#c3c6ec5ee6125c6993c5cbca20dc8621a9ea7a6e"
+ integrity sha512-kLs1j9Nn4MQoBYdRXH6AeaXMbEJFaFu/v1nQkvib6QzTj8MZI5OQzqmD83/2jEM1z0DLilra5aWO5YpyC0ALIw==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-create-regexp-features-plugin" "^7.8.3"
+ "@babel/helper-plugin-utils" "^7.8.3"
-"@babel/plugin-transform-destructuring@^7.4.3", "@babel/plugin-transform-destructuring@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.7.4.tgz#2b713729e5054a1135097b6a67da1b6fe8789267"
- integrity sha512-4jFMXI1Cu2aXbcXXl8Lr6YubCn6Oc7k9lLsu8v61TZh+1jny2BWmdtvY9zSUlLdGUvcy9DMAWyZEOqjsbeg/wA==
+"@babel/plugin-transform-duplicate-keys@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-duplicate-keys/-/plugin-transform-duplicate-keys-7.8.3.tgz#8d12df309aa537f272899c565ea1768e286e21f1"
+ integrity sha512-s8dHiBUbcbSgipS4SMFuWGqCvyge5V2ZeAWzR6INTVC3Ltjig/Vw1G2Gztv0vU/hRG9X8IvKvYdoksnUfgXOEQ==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
-"@babel/plugin-transform-dotall-regex@^7.4.3", "@babel/plugin-transform-dotall-regex@^7.7.7":
- version "7.7.7"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.7.7.tgz#3e9713f1b69f339e87fa796b097d73ded16b937b"
- integrity sha512-b4in+YlTeE/QmTgrllnb3bHA0HntYvjz8O3Mcbx75UBPJA2xhb5A8nle498VhxSXJHQefjtQxpnLPehDJ4TRlg==
+"@babel/plugin-transform-exponentiation-operator@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-exponentiation-operator/-/plugin-transform-exponentiation-operator-7.8.3.tgz#581a6d7f56970e06bf51560cd64f5e947b70d7b7"
+ integrity sha512-zwIpuIymb3ACcInbksHaNcR12S++0MDLKkiqXHl3AzpgdKlFNhog+z/K0+TGW+b0w5pgTq4H6IwV/WhxbGYSjQ==
dependencies:
- "@babel/helper-create-regexp-features-plugin" "^7.7.4"
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-builder-binary-assignment-operator-visitor" "^7.8.3"
+ "@babel/helper-plugin-utils" "^7.8.3"
-"@babel/plugin-transform-duplicate-keys@^7.2.0", "@babel/plugin-transform-duplicate-keys@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-duplicate-keys/-/plugin-transform-duplicate-keys-7.7.4.tgz#3d21731a42e3f598a73835299dd0169c3b90ac91"
- integrity sha512-g1y4/G6xGWMD85Tlft5XedGaZBCIVN+/P0bs6eabmcPP9egFleMAo65OOjlhcz1njpwagyY3t0nsQC9oTFegJA==
+"@babel/plugin-transform-flow-strip-types@7.9.0":
+ version "7.9.0"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-flow-strip-types/-/plugin-transform-flow-strip-types-7.9.0.tgz#8a3538aa40434e000b8f44a3c5c9ac7229bd2392"
+ integrity sha512-7Qfg0lKQhEHs93FChxVLAvhBshOPQDtJUTVHr/ZwQNRccCm4O9D79r9tVSoV8iNwjP1YgfD+e/fgHcPkN1qEQg==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
+ "@babel/plugin-syntax-flow" "^7.8.3"
-"@babel/plugin-transform-exponentiation-operator@^7.2.0", "@babel/plugin-transform-exponentiation-operator@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-exponentiation-operator/-/plugin-transform-exponentiation-operator-7.7.4.tgz#dd30c0191e3a1ba19bcc7e389bdfddc0729d5db9"
- integrity sha512-MCqiLfCKm6KEA1dglf6Uqq1ElDIZwFuzz1WH5mTf8k2uQSxEJMbOIEh7IZv7uichr7PMfi5YVSrr1vz+ipp7AQ==
+"@babel/plugin-transform-for-of@^7.9.0":
+ version "7.9.0"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-for-of/-/plugin-transform-for-of-7.9.0.tgz#0f260e27d3e29cd1bb3128da5e76c761aa6c108e"
+ integrity sha512-lTAnWOpMwOXpyDx06N+ywmF3jNbafZEqZ96CGYabxHrxNX8l5ny7dt4bK/rGwAh9utyP2b2Hv7PlZh1AAS54FQ==
dependencies:
- "@babel/helper-builder-binary-assignment-operator-visitor" "^7.7.4"
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
-"@babel/plugin-transform-flow-strip-types@7.4.0":
- version "7.4.0"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-flow-strip-types/-/plugin-transform-flow-strip-types-7.4.0.tgz#f3c59eecff68c99b9c96eaafe4fe9d1fa8947138"
- integrity sha512-C4ZVNejHnfB22vI2TYN4RUp2oCmq6cSEAg4RygSvYZUECRqUu9O4PMEMNJ4wsemaRGg27BbgYctG4BZh+AgIHw==
+"@babel/plugin-transform-function-name@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-function-name/-/plugin-transform-function-name-7.8.3.tgz#279373cb27322aaad67c2683e776dfc47196ed8b"
+ integrity sha512-rO/OnDS78Eifbjn5Py9v8y0aR+aSYhDhqAwVfsTl0ERuMZyr05L1aFSCJnbv2mmsLkit/4ReeQ9N2BgLnOcPCQ==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
- "@babel/plugin-syntax-flow" "^7.2.0"
+ "@babel/helper-function-name" "^7.8.3"
+ "@babel/helper-plugin-utils" "^7.8.3"
-"@babel/plugin-transform-for-of@^7.4.3", "@babel/plugin-transform-for-of@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-for-of/-/plugin-transform-for-of-7.7.4.tgz#248800e3a5e507b1f103d8b4ca998e77c63932bc"
- integrity sha512-zZ1fD1B8keYtEcKF+M1TROfeHTKnijcVQm0yO/Yu1f7qoDoxEIc/+GX6Go430Bg84eM/xwPFp0+h4EbZg7epAA==
+"@babel/plugin-transform-literals@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-literals/-/plugin-transform-literals-7.8.3.tgz#aef239823d91994ec7b68e55193525d76dbd5dc1"
+ integrity sha512-3Tqf8JJ/qB7TeldGl+TT55+uQei9JfYaregDcEAyBZ7akutriFrt6C/wLYIer6OYhleVQvH/ntEhjE/xMmy10A==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
-"@babel/plugin-transform-function-name@^7.4.3", "@babel/plugin-transform-function-name@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-function-name/-/plugin-transform-function-name-7.7.4.tgz#75a6d3303d50db638ff8b5385d12451c865025b1"
- integrity sha512-E/x09TvjHNhsULs2IusN+aJNRV5zKwxu1cpirZyRPw+FyyIKEHPXTsadj48bVpc1R5Qq1B5ZkzumuFLytnbT6g==
+"@babel/plugin-transform-member-expression-literals@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-member-expression-literals/-/plugin-transform-member-expression-literals-7.8.3.tgz#963fed4b620ac7cbf6029c755424029fa3a40410"
+ integrity sha512-3Wk2EXhnw+rP+IDkK6BdtPKsUE5IeZ6QOGrPYvw52NwBStw9V1ZVzxgK6fSKSxqUvH9eQPR3tm3cOq79HlsKYA==
dependencies:
- "@babel/helper-function-name" "^7.7.4"
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
-"@babel/plugin-transform-literals@^7.2.0", "@babel/plugin-transform-literals@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-literals/-/plugin-transform-literals-7.7.4.tgz#27fe87d2b5017a2a5a34d1c41a6b9f6a6262643e"
- integrity sha512-X2MSV7LfJFm4aZfxd0yLVFrEXAgPqYoDG53Br/tCKiKYfX0MjVjQeWPIhPHHsCqzwQANq+FLN786fF5rgLS+gw==
+"@babel/plugin-transform-modules-amd@^7.9.0", "@babel/plugin-transform-modules-amd@^7.9.6":
+ version "7.9.6"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-amd/-/plugin-transform-modules-amd-7.9.6.tgz#8539ec42c153d12ea3836e0e3ac30d5aae7b258e"
+ integrity sha512-zoT0kgC3EixAyIAU+9vfaUVKTv9IxBDSabgHoUCBP6FqEJ+iNiN7ip7NBKcYqbfUDfuC2mFCbM7vbu4qJgOnDw==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-module-transforms" "^7.9.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
+ babel-plugin-dynamic-import-node "^2.3.3"
-"@babel/plugin-transform-member-expression-literals@^7.2.0", "@babel/plugin-transform-member-expression-literals@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-member-expression-literals/-/plugin-transform-member-expression-literals-7.7.4.tgz#aee127f2f3339fc34ce5e3055d7ffbf7aa26f19a"
- integrity sha512-9VMwMO7i69LHTesL0RdGy93JU6a+qOPuvB4F4d0kR0zyVjJRVJRaoaGjhtki6SzQUu8yen/vxPKN6CWnCUw6bA==
+"@babel/plugin-transform-modules-commonjs@^7.9.0", "@babel/plugin-transform-modules-commonjs@^7.9.6":
+ version "7.9.6"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.9.6.tgz#64b7474a4279ee588cacd1906695ca721687c277"
+ integrity sha512-7H25fSlLcn+iYimmsNe3uK1at79IE6SKW9q0/QeEHTMC9MdOZ+4bA+T1VFB5fgOqBWoqlifXRzYD0JPdmIrgSQ==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-module-transforms" "^7.9.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
+ "@babel/helper-simple-access" "^7.8.3"
+ babel-plugin-dynamic-import-node "^2.3.3"
-"@babel/plugin-transform-modules-amd@^7.2.0", "@babel/plugin-transform-modules-amd@^7.7.5":
- version "7.7.5"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-amd/-/plugin-transform-modules-amd-7.7.5.tgz#39e0fb717224b59475b306402bb8eedab01e729c"
- integrity sha512-CT57FG4A2ZUNU1v+HdvDSDrjNWBrtCmSH6YbbgN3Lrf0Di/q/lWRxZrE72p3+HCCz9UjfZOEBdphgC0nzOS6DQ==
+"@babel/plugin-transform-modules-systemjs@^7.9.0", "@babel/plugin-transform-modules-systemjs@^7.9.6":
+ version "7.9.6"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.9.6.tgz#207f1461c78a231d5337a92140e52422510d81a4"
+ integrity sha512-NW5XQuW3N2tTHim8e1b7qGy7s0kZ2OH3m5octc49K1SdAKGxYxeIx7hiIz05kS1R2R+hOWcsr1eYwcGhrdHsrg==
dependencies:
- "@babel/helper-module-transforms" "^7.7.5"
- "@babel/helper-plugin-utils" "^7.0.0"
- babel-plugin-dynamic-import-node "^2.3.0"
+ "@babel/helper-hoist-variables" "^7.8.3"
+ "@babel/helper-module-transforms" "^7.9.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
+ babel-plugin-dynamic-import-node "^2.3.3"
-"@babel/plugin-transform-modules-commonjs@^7.4.3", "@babel/plugin-transform-modules-commonjs@^7.7.5":
- version "7.7.5"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.7.5.tgz#1d27f5eb0bcf7543e774950e5b2fa782e637b345"
- integrity sha512-9Cq4zTFExwFhQI6MT1aFxgqhIsMWQWDVwOgLzl7PTWJHsNaqFvklAU+Oz6AQLAS0dJKTwZSOCo20INwktxpi3Q==
+"@babel/plugin-transform-modules-umd@^7.9.0":
+ version "7.9.0"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-umd/-/plugin-transform-modules-umd-7.9.0.tgz#e909acae276fec280f9b821a5f38e1f08b480697"
+ integrity sha512-uTWkXkIVtg/JGRSIABdBoMsoIeoHQHPTL0Y2E7xf5Oj7sLqwVsNXOkNk0VJc7vF0IMBsPeikHxFjGe+qmwPtTQ==
dependencies:
- "@babel/helper-module-transforms" "^7.7.5"
- "@babel/helper-plugin-utils" "^7.0.0"
- "@babel/helper-simple-access" "^7.7.4"
- babel-plugin-dynamic-import-node "^2.3.0"
+ "@babel/helper-module-transforms" "^7.9.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
-"@babel/plugin-transform-modules-systemjs@^7.4.0", "@babel/plugin-transform-modules-systemjs@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.7.4.tgz#cd98152339d3e763dfe838b7d4273edaf520bb30"
- integrity sha512-y2c96hmcsUi6LrMqvmNDPBBiGCiQu0aYqpHatVVu6kD4mFEXKjyNxd/drc18XXAf9dv7UXjrZwBVmTTGaGP8iw==
+"@babel/plugin-transform-named-capturing-groups-regex@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.8.3.tgz#a2a72bffa202ac0e2d0506afd0939c5ecbc48c6c"
+ integrity sha512-f+tF/8UVPU86TrCb06JoPWIdDpTNSGGcAtaD9mLP0aYGA0OS0j7j7DHJR0GTFrUZPUU6loZhbsVZgTh0N+Qdnw==
dependencies:
- "@babel/helper-hoist-variables" "^7.7.4"
- "@babel/helper-plugin-utils" "^7.0.0"
- babel-plugin-dynamic-import-node "^2.3.0"
+ "@babel/helper-create-regexp-features-plugin" "^7.8.3"
-"@babel/plugin-transform-modules-umd@^7.2.0", "@babel/plugin-transform-modules-umd@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-umd/-/plugin-transform-modules-umd-7.7.4.tgz#1027c355a118de0aae9fee00ad7813c584d9061f"
- integrity sha512-u2B8TIi0qZI4j8q4C51ktfO7E3cQ0qnaXFI1/OXITordD40tt17g/sXqgNNCcMTcBFKrUPcGDx+TBJuZxLx7tw==
+"@babel/plugin-transform-new-target@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-new-target/-/plugin-transform-new-target-7.8.3.tgz#60cc2ae66d85c95ab540eb34babb6434d4c70c43"
+ integrity sha512-QuSGysibQpyxexRyui2vca+Cmbljo8bcRckgzYV4kRIsHpVeyeC3JDO63pY+xFZ6bWOBn7pfKZTqV4o/ix9sFw==
dependencies:
- "@babel/helper-module-transforms" "^7.7.4"
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
-"@babel/plugin-transform-named-capturing-groups-regex@^7.4.2", "@babel/plugin-transform-named-capturing-groups-regex@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.7.4.tgz#fb3bcc4ee4198e7385805007373d6b6f42c98220"
- integrity sha512-jBUkiqLKvUWpv9GLSuHUFYdmHg0ujC1JEYoZUfeOOfNydZXp1sXObgyPatpcwjWgsdBGsagWW0cdJpX/DO2jMw==
+"@babel/plugin-transform-object-super@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-object-super/-/plugin-transform-object-super-7.8.3.tgz#ebb6a1e7a86ffa96858bd6ac0102d65944261725"
+ integrity sha512-57FXk+gItG/GejofIyLIgBKTas4+pEU47IXKDBWFTxdPd7F80H8zybyAY7UoblVfBhBGs2EKM+bJUu2+iUYPDQ==
dependencies:
- "@babel/helper-create-regexp-features-plugin" "^7.7.4"
+ "@babel/helper-plugin-utils" "^7.8.3"
+ "@babel/helper-replace-supers" "^7.8.3"
-"@babel/plugin-transform-new-target@^7.4.0", "@babel/plugin-transform-new-target@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-new-target/-/plugin-transform-new-target-7.7.4.tgz#4a0753d2d60639437be07b592a9e58ee00720167"
- integrity sha512-CnPRiNtOG1vRodnsyGX37bHQleHE14B9dnnlgSeEs3ek3fHN1A1SScglTCg1sfbe7sRQ2BUcpgpTpWSfMKz3gg==
+"@babel/plugin-transform-parameters@^7.8.7", "@babel/plugin-transform-parameters@^7.9.5":
+ version "7.9.5"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.9.5.tgz#173b265746f5e15b2afe527eeda65b73623a0795"
+ integrity sha512-0+1FhHnMfj6lIIhVvS4KGQJeuhe1GI//h5uptK4PvLt+BGBxsoUJbd3/IW002yk//6sZPlFgsG1hY6OHLcy6kA==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-get-function-arity" "^7.8.3"
+ "@babel/helper-plugin-utils" "^7.8.3"
-"@babel/plugin-transform-object-super@^7.2.0", "@babel/plugin-transform-object-super@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-object-super/-/plugin-transform-object-super-7.7.4.tgz#48488937a2d586c0148451bf51af9d7dda567262"
- integrity sha512-ho+dAEhC2aRnff2JCA0SAK7V2R62zJd/7dmtoe7MHcso4C2mS+vZjn1Pb1pCVZvJs1mgsvv5+7sT+m3Bysb6eg==
+"@babel/plugin-transform-property-literals@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-property-literals/-/plugin-transform-property-literals-7.8.3.tgz#33194300d8539c1ed28c62ad5087ba3807b98263"
+ integrity sha512-uGiiXAZMqEoQhRWMK17VospMZh5sXWg+dlh2soffpkAl96KAm+WZuJfa6lcELotSRmooLqg0MWdH6UUq85nmmg==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
- "@babel/helper-replace-supers" "^7.7.4"
-
-"@babel/plugin-transform-parameters@^7.4.3", "@babel/plugin-transform-parameters@^7.7.7":
- version "7.7.7"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.7.7.tgz#7a884b2460164dc5f194f668332736584c760007"
- integrity sha512-OhGSrf9ZBrr1fw84oFXj5hgi8Nmg+E2w5L7NhnG0lPvpDtqd7dbyilM2/vR8CKbJ907RyxPh2kj6sBCSSfI9Ew==
- dependencies:
- "@babel/helper-call-delegate" "^7.7.4"
- "@babel/helper-get-function-arity" "^7.7.4"
- "@babel/helper-plugin-utils" "^7.0.0"
-
-"@babel/plugin-transform-property-literals@^7.2.0", "@babel/plugin-transform-property-literals@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-property-literals/-/plugin-transform-property-literals-7.7.4.tgz#2388d6505ef89b266103f450f9167e6bd73f98c2"
- integrity sha512-MatJhlC4iHsIskWYyawl53KuHrt+kALSADLQQ/HkhTjX954fkxIEh4q5slL4oRAnsm/eDoZ4q0CIZpcqBuxhJQ==
- dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
-
-"@babel/plugin-transform-react-constant-elements@7.2.0":
- version "7.2.0"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-constant-elements/-/plugin-transform-react-constant-elements-7.2.0.tgz#ed602dc2d8bff2f0cb1a5ce29263dbdec40779f7"
- integrity sha512-YYQFg6giRFMsZPKUM9v+VcHOdfSQdz9jHCx3akAi3UYgyjndmdYGSXylQ/V+HswQt4fL8IklchD9HTsaOCrWQQ==
- dependencies:
- "@babel/helper-annotate-as-pure" "^7.0.0"
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
"@babel/plugin-transform-react-constant-elements@^7.0.0":
version "7.7.4"
@@ -674,21 +809,30 @@
"@babel/helper-annotate-as-pure" "^7.7.4"
"@babel/helper-plugin-utils" "^7.0.0"
-"@babel/plugin-transform-react-display-name@7.2.0":
- version "7.2.0"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-display-name/-/plugin-transform-react-display-name-7.2.0.tgz#ebfaed87834ce8dc4279609a4f0c324c156e3eb0"
- integrity sha512-Htf/tPa5haZvRMiNSQSFifK12gtr/8vwfr+A9y69uF0QcU77AVu4K7MiHEkTxF7lQoHOL0F9ErqgfNEAKgXj7A==
+"@babel/plugin-transform-react-display-name@7.8.3", "@babel/plugin-transform-react-display-name@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-display-name/-/plugin-transform-react-display-name-7.8.3.tgz#70ded987c91609f78353dd76d2fb2a0bb991e8e5"
+ integrity sha512-3Jy/PCw8Fe6uBKtEgz3M82ljt+lTg+xJaM4og+eyu83qLT87ZUSckn0wy7r31jflURWLO83TW6Ylf7lyXj3m5A==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
-"@babel/plugin-transform-react-display-name@^7.0.0", "@babel/plugin-transform-react-display-name@^7.7.4":
+"@babel/plugin-transform-react-display-name@^7.7.4":
version "7.7.4"
resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-display-name/-/plugin-transform-react-display-name-7.7.4.tgz#9f2b80b14ebc97eef4a9b29b612c58ed9c0d10dd"
integrity sha512-sBbIvqYkthai0X0vkD2xsAwluBp+LtNHH+/V4a5ydifmTtb8KOVOlrMIk/MYmIc4uTYDnjZUHQildYNo36SRJw==
dependencies:
"@babel/helper-plugin-utils" "^7.0.0"
-"@babel/plugin-transform-react-jsx-self@^7.0.0", "@babel/plugin-transform-react-jsx-self@^7.7.4":
+"@babel/plugin-transform-react-jsx-development@^7.9.0":
+ version "7.9.0"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx-development/-/plugin-transform-react-jsx-development-7.9.0.tgz#3c2a130727caf00c2a293f0aed24520825dbf754"
+ integrity sha512-tK8hWKrQncVvrhvtOiPpKrQjfNX3DtkNLSX4ObuGcpS9p0QrGetKmlySIGR07y48Zft8WVgPakqd/bk46JrMSw==
+ dependencies:
+ "@babel/helper-builder-react-jsx-experimental" "^7.9.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
+ "@babel/plugin-syntax-jsx" "^7.8.3"
+
+"@babel/plugin-transform-react-jsx-self@^7.7.4":
version "7.7.4"
resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.7.4.tgz#81b8fbfd14b2215e8f1c2c3adfba266127b0231c"
integrity sha512-PWYjSfqrO273mc1pKCRTIJXyqfc9vWYBax88yIhQb+bpw3XChVC7VWS4VwRVs63wFHKxizvGSd00XEr+YB9Q2A==
@@ -696,7 +840,15 @@
"@babel/helper-plugin-utils" "^7.0.0"
"@babel/plugin-syntax-jsx" "^7.7.4"
-"@babel/plugin-transform-react-jsx-source@^7.0.0", "@babel/plugin-transform-react-jsx-source@^7.7.4":
+"@babel/plugin-transform-react-jsx-self@^7.9.0":
+ version "7.9.0"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.9.0.tgz#f4f26a325820205239bb915bad8e06fcadabb49b"
+ integrity sha512-K2ObbWPKT7KUTAoyjCsFilOkEgMvFG+y0FqOl6Lezd0/13kMkkjHskVsZvblRPj1PHA44PrToaZANrryppzTvQ==
+ dependencies:
+ "@babel/helper-plugin-utils" "^7.8.3"
+ "@babel/plugin-syntax-jsx" "^7.8.3"
+
+"@babel/plugin-transform-react-jsx-source@^7.7.4":
version "7.7.4"
resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.7.4.tgz#8994b1bf6014b133f5a46d3b7d1ee5f5e3e72c10"
integrity sha512-5ZU9FnPhqtHsOXxutRtXZAzoEJwDaP32QcobbMP1/qt7NYcsCNK8XgzJcJfoEr/ZnzVvUNInNjIW22Z6I8p9mg==
@@ -704,7 +856,15 @@
"@babel/helper-plugin-utils" "^7.0.0"
"@babel/plugin-syntax-jsx" "^7.7.4"
-"@babel/plugin-transform-react-jsx@^7.0.0", "@babel/plugin-transform-react-jsx@^7.7.4":
+"@babel/plugin-transform-react-jsx-source@^7.9.0":
+ version "7.9.0"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.9.0.tgz#89ef93025240dd5d17d3122294a093e5e0183de0"
+ integrity sha512-K6m3LlSnTSfRkM6FcRk8saNEeaeyG5k7AVkBU2bZK3+1zdkSED3qNdsWrUgQBeTVD2Tp3VMmerxVO2yM5iITmw==
+ dependencies:
+ "@babel/helper-plugin-utils" "^7.8.3"
+ "@babel/plugin-syntax-jsx" "^7.8.3"
+
+"@babel/plugin-transform-react-jsx@^7.7.4":
version "7.7.7"
resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.7.7.tgz#5cbaa7445b4a09f774029f3cc7bb448ff3122a5d"
integrity sha512-SlPjWPbva2+7/ZJbGcoqjl4LsQaLpKEzxW9hcxU7675s24JmdotJOSJ4cgAbV82W3FcZpHIGmRZIlUL8ayMvjw==
@@ -713,205 +873,248 @@
"@babel/helper-plugin-utils" "^7.0.0"
"@babel/plugin-syntax-jsx" "^7.7.4"
-"@babel/plugin-transform-regenerator@^7.4.3", "@babel/plugin-transform-regenerator@^7.7.5":
- version "7.7.5"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.7.5.tgz#3a8757ee1a2780f390e89f246065ecf59c26fce9"
- integrity sha512-/8I8tPvX2FkuEyWbjRCt4qTAgZK0DVy8QRguhA524UH48RfGJy94On2ri+dCuwOpcerPRl9O4ebQkRcVzIaGBw==
+"@babel/plugin-transform-react-jsx@^7.9.1":
+ version "7.9.4"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.9.4.tgz#86f576c8540bd06d0e95e0b61ea76d55f6cbd03f"
+ integrity sha512-Mjqf3pZBNLt854CK0C/kRuXAnE6H/bo7xYojP+WGtX8glDGSibcwnsWwhwoSuRg0+EBnxPC1ouVnuetUIlPSAw==
dependencies:
- regenerator-transform "^0.14.0"
+ "@babel/helper-builder-react-jsx" "^7.9.0"
+ "@babel/helper-builder-react-jsx-experimental" "^7.9.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
+ "@babel/plugin-syntax-jsx" "^7.8.3"
-"@babel/plugin-transform-reserved-words@^7.2.0", "@babel/plugin-transform-reserved-words@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-reserved-words/-/plugin-transform-reserved-words-7.7.4.tgz#6a7cf123ad175bb5c69aec8f6f0770387ed3f1eb"
- integrity sha512-OrPiUB5s5XvkCO1lS7D8ZtHcswIC57j62acAnJZKqGGnHP+TIc/ljQSrgdX/QyOTdEK5COAhuc820Hi1q2UgLQ==
+"@babel/plugin-transform-regenerator@^7.8.7":
+ version "7.8.7"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.8.7.tgz#5e46a0dca2bee1ad8285eb0527e6abc9c37672f8"
+ integrity sha512-TIg+gAl4Z0a3WmD3mbYSk+J9ZUH6n/Yc57rtKRnlA/7rcCvpekHXe0CMZHP1gYp7/KLe9GHTuIba0vXmls6drA==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
+ regenerator-transform "^0.14.2"
-"@babel/plugin-transform-runtime@7.4.3":
- version "7.4.3"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-runtime/-/plugin-transform-runtime-7.4.3.tgz#4d6691690ecdc9f5cb8c3ab170a1576c1f556371"
- integrity sha512-7Q61bU+uEI7bCUFReT1NKn7/X6sDQsZ7wL1sJ9IYMAO7cI+eg6x9re1cEw2fCRMbbTVyoeUKWSV1M6azEfKCfg==
+"@babel/plugin-transform-reserved-words@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-reserved-words/-/plugin-transform-reserved-words-7.8.3.tgz#9a0635ac4e665d29b162837dd3cc50745dfdf1f5"
+ integrity sha512-mwMxcycN3omKFDjDQUl+8zyMsBfjRFr0Zn/64I41pmjv4NJuqcYlEtezwYtw9TFd9WR1vN5kiM+O0gMZzO6L0A==
dependencies:
- "@babel/helper-module-imports" "^7.0.0"
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
+
+"@babel/plugin-transform-runtime@7.9.0":
+ version "7.9.0"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-runtime/-/plugin-transform-runtime-7.9.0.tgz#45468c0ae74cc13204e1d3b1f4ce6ee83258af0b"
+ integrity sha512-pUu9VSf3kI1OqbWINQ7MaugnitRss1z533436waNXp+0N3ur3zfut37sXiQMxkuCF4VUjwZucen/quskCh7NHw==
+ dependencies:
+ "@babel/helper-module-imports" "^7.8.3"
+ "@babel/helper-plugin-utils" "^7.8.3"
resolve "^1.8.1"
semver "^5.5.1"
-"@babel/plugin-transform-shorthand-properties@^7.2.0", "@babel/plugin-transform-shorthand-properties@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.7.4.tgz#74a0a9b2f6d67a684c6fbfd5f0458eb7ba99891e"
- integrity sha512-q+suddWRfIcnyG5YiDP58sT65AJDZSUhXQDZE3r04AuqD6d/XLaQPPXSBzP2zGerkgBivqtQm9XKGLuHqBID6Q==
+"@babel/plugin-transform-shorthand-properties@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.8.3.tgz#28545216e023a832d4d3a1185ed492bcfeac08c8"
+ integrity sha512-I9DI6Odg0JJwxCHzbzW08ggMdCezoWcuQRz3ptdudgwaHxTjxw5HgdFJmZIkIMlRymL6YiZcped4TTCB0JcC8w==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
-"@babel/plugin-transform-spread@^7.2.0", "@babel/plugin-transform-spread@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-spread/-/plugin-transform-spread-7.7.4.tgz#aa673b356fe6b7e70d69b6e33a17fef641008578"
- integrity sha512-8OSs0FLe5/80cndziPlg4R0K6HcWSM0zyNhHhLsmw/Nc5MaA49cAsnoJ/t/YZf8qkG7fD+UjTRaApVDB526d7Q==
+"@babel/plugin-transform-spread@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-spread/-/plugin-transform-spread-7.8.3.tgz#9c8ffe8170fdfb88b114ecb920b82fb6e95fe5e8"
+ integrity sha512-CkuTU9mbmAoFOI1tklFWYYbzX5qCIZVXPVy0jpXgGwkplCndQAa58s2jr66fTeQnA64bDox0HL4U56CFYoyC7g==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
-"@babel/plugin-transform-sticky-regex@^7.2.0", "@babel/plugin-transform-sticky-regex@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-sticky-regex/-/plugin-transform-sticky-regex-7.7.4.tgz#ffb68c05090c30732076b1285dc1401b404a123c"
- integrity sha512-Ls2NASyL6qtVe1H1hXts9yuEeONV2TJZmplLONkMPUG158CtmnrzW5Q5teibM5UVOFjG0D3IC5mzXR6pPpUY7A==
+"@babel/plugin-transform-sticky-regex@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-sticky-regex/-/plugin-transform-sticky-regex-7.8.3.tgz#be7a1290f81dae767475452199e1f76d6175b100"
+ integrity sha512-9Spq0vGCD5Bb4Z/ZXXSK5wbbLFMG085qd2vhL1JYu1WcQ5bXqZBAYRzU1d+p79GcHs2szYv5pVQCX13QgldaWw==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
- "@babel/helper-regex" "^7.0.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
+ "@babel/helper-regex" "^7.8.3"
-"@babel/plugin-transform-template-literals@^7.2.0", "@babel/plugin-transform-template-literals@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.7.4.tgz#1eb6411736dd3fe87dbd20cc6668e5121c17d604"
- integrity sha512-sA+KxLwF3QwGj5abMHkHgshp9+rRz+oY9uoRil4CyLtgEuE/88dpkeWgNk5qKVsJE9iSfly3nvHapdRiIS2wnQ==
+"@babel/plugin-transform-template-literals@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.8.3.tgz#7bfa4732b455ea6a43130adc0ba767ec0e402a80"
+ integrity sha512-820QBtykIQOLFT8NZOcTRJ1UNuztIELe4p9DCgvj4NK+PwluSJ49we7s9FB1HIGNIYT7wFUJ0ar2QpCDj0escQ==
dependencies:
- "@babel/helper-annotate-as-pure" "^7.7.4"
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-annotate-as-pure" "^7.8.3"
+ "@babel/helper-plugin-utils" "^7.8.3"
-"@babel/plugin-transform-typeof-symbol@^7.2.0", "@babel/plugin-transform-typeof-symbol@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-typeof-symbol/-/plugin-transform-typeof-symbol-7.7.4.tgz#3174626214f2d6de322882e498a38e8371b2140e"
- integrity sha512-KQPUQ/7mqe2m0B8VecdyaW5XcQYaePyl9R7IsKd+irzj6jvbhoGnRE+M0aNkyAzI07VfUQ9266L5xMARitV3wg==
+"@babel/plugin-transform-typeof-symbol@^7.8.4":
+ version "7.8.4"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-typeof-symbol/-/plugin-transform-typeof-symbol-7.8.4.tgz#ede4062315ce0aaf8a657a920858f1a2f35fc412"
+ integrity sha512-2QKyfjGdvuNfHsb7qnBBlKclbD4CfshH2KvDabiijLMGXPHJXGxtDzwIF7bQP+T0ysw8fYTtxPafgfs/c1Lrqg==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-plugin-utils" "^7.8.3"
-"@babel/plugin-transform-typescript@^7.3.2":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-typescript/-/plugin-transform-typescript-7.7.4.tgz#2974fd05f4e85c695acaf497f432342de9fc0636"
- integrity sha512-X8e3tcPEKnwwPVG+vP/vSqEShkwODOEeyQGod82qrIuidwIrfnsGn11qPM1jBLF4MqguTXXYzm58d0dY+/wdpg==
+"@babel/plugin-transform-typescript@^7.9.0":
+ version "7.9.6"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-typescript/-/plugin-transform-typescript-7.9.6.tgz#2248971416a506fc78278fc0c0ea3179224af1e9"
+ integrity sha512-8OvsRdvpt3Iesf2qsAn+YdlwAJD7zJ+vhFZmDCa4b8dTp7MmHtKk5FF2mCsGxjZwuwsy/yIIay/nLmxST1ctVQ==
dependencies:
- "@babel/helper-create-class-features-plugin" "^7.7.4"
- "@babel/helper-plugin-utils" "^7.0.0"
- "@babel/plugin-syntax-typescript" "^7.7.4"
+ "@babel/helper-create-class-features-plugin" "^7.9.6"
+ "@babel/helper-plugin-utils" "^7.8.3"
+ "@babel/plugin-syntax-typescript" "^7.8.3"
-"@babel/plugin-transform-unicode-regex@^7.4.3", "@babel/plugin-transform-unicode-regex@^7.7.4":
- version "7.7.4"
- resolved "https://registry.yarnpkg.com/@babel/plugin-transform-unicode-regex/-/plugin-transform-unicode-regex-7.7.4.tgz#a3c0f65b117c4c81c5b6484f2a5e7b95346b83ae"
- integrity sha512-N77UUIV+WCvE+5yHw+oks3m18/umd7y392Zv7mYTpFqHtkpcc+QUz+gLJNTWVlWROIWeLqY0f3OjZxV5TcXnRw==
+"@babel/plugin-transform-unicode-regex@^7.8.3":
+ version "7.8.3"
+ resolved "https://registry.yarnpkg.com/@babel/plugin-transform-unicode-regex/-/plugin-transform-unicode-regex-7.8.3.tgz#0cef36e3ba73e5c57273effb182f46b91a1ecaad"
+ integrity sha512-+ufgJjYdmWfSQ+6NS9VGUR2ns8cjJjYbrbi11mZBTaWm+Fui/ncTLFF28Ei1okavY+xkojGr1eJxNsWYeA5aZw==
dependencies:
- "@babel/helper-create-regexp-features-plugin" "^7.7.4"
- "@babel/helper-plugin-utils" "^7.0.0"
+ "@babel/helper-create-regexp-features-plugin" "^7.8.3"
+ "@babel/helper-plugin-utils" "^7.8.3"
-"@babel/preset-env@7.4.3":
- version "7.4.3"
- resolved "https://registry.yarnpkg.com/@babel/preset-env/-/preset-env-7.4.3.tgz#e71e16e123dc0fbf65a52cbcbcefd072fbd02880"
- integrity sha512-FYbZdV12yHdJU5Z70cEg0f6lvtpZ8jFSDakTm7WXeJbLXh4R0ztGEu/SW7G1nJ2ZvKwDhz8YrbA84eYyprmGqw==
+"@babel/preset-env@7.9.0":
+ version "7.9.0"
+ resolved "https://registry.yarnpkg.com/@babel/preset-env/-/preset-env-7.9.0.tgz#a5fc42480e950ae8f5d9f8f2bbc03f52722df3a8"
+ integrity sha512-712DeRXT6dyKAM/FMbQTV/FvRCms2hPCx+3weRjZ8iQVQWZejWWk1wwG6ViWMyqb/ouBbGOl5b6aCk0+j1NmsQ==
dependencies:
- "@babel/helper-module-imports" "^7.0.0"
- "@babel/helper-plugin-utils" "^7.0.0"
- "@babel/plugin-proposal-async-generator-functions" "^7.2.0"
- "@babel/plugin-proposal-json-strings" "^7.2.0"
- "@babel/plugin-proposal-object-rest-spread" "^7.4.3"
- "@babel/plugin-proposal-optional-catch-binding" "^7.2.0"
- "@babel/plugin-proposal-unicode-property-regex" "^7.4.0"
- "@babel/plugin-syntax-async-generators" "^7.2.0"
- "@babel/plugin-syntax-json-strings" "^7.2.0"
- "@babel/plugin-syntax-object-rest-spread" "^7.2.0"
- "@babel/plugin-syntax-optional-catch-binding" "^7.2.0"
- "@babel/plugin-transform-arrow-functions" "^7.2.0"
- "@babel/plugin-transform-async-to-generator" "^7.4.0"
- "@babel/plugin-transform-block-scoped-functions" "^7.2.0"
- "@babel/plugin-transform-block-scoping" "^7.4.0"
- "@babel/plugin-transform-classes" "^7.4.3"
- "@babel/plugin-transform-computed-properties" "^7.2.0"
- "@babel/plugin-transform-destructuring" "^7.4.3"
- "@babel/plugin-transform-dotall-regex" "^7.4.3"
- "@babel/plugin-transform-duplicate-keys" "^7.2.0"
- "@babel/plugin-transform-exponentiation-operator" "^7.2.0"
- "@babel/plugin-transform-for-of" "^7.4.3"
- "@babel/plugin-transform-function-name" "^7.4.3"
- "@babel/plugin-transform-literals" "^7.2.0"
- "@babel/plugin-transform-member-expression-literals" "^7.2.0"
- "@babel/plugin-transform-modules-amd" "^7.2.0"
- "@babel/plugin-transform-modules-commonjs" "^7.4.3"
- "@babel/plugin-transform-modules-systemjs" "^7.4.0"
- "@babel/plugin-transform-modules-umd" "^7.2.0"
- "@babel/plugin-transform-named-capturing-groups-regex" "^7.4.2"
- "@babel/plugin-transform-new-target" "^7.4.0"
- "@babel/plugin-transform-object-super" "^7.2.0"
- "@babel/plugin-transform-parameters" "^7.4.3"
- "@babel/plugin-transform-property-literals" "^7.2.0"
- "@babel/plugin-transform-regenerator" "^7.4.3"
- "@babel/plugin-transform-reserved-words" "^7.2.0"
- "@babel/plugin-transform-shorthand-properties" "^7.2.0"
- "@babel/plugin-transform-spread" "^7.2.0"
- "@babel/plugin-transform-sticky-regex" "^7.2.0"
- "@babel/plugin-transform-template-literals" "^7.2.0"
- "@babel/plugin-transform-typeof-symbol" "^7.2.0"
- "@babel/plugin-transform-unicode-regex" "^7.4.3"
- "@babel/types" "^7.4.0"
- browserslist "^4.5.2"
- core-js-compat "^3.0.0"
+ "@babel/compat-data" "^7.9.0"
+ "@babel/helper-compilation-targets" "^7.8.7"
+ "@babel/helper-module-imports" "^7.8.3"
+ "@babel/helper-plugin-utils" "^7.8.3"
+ "@babel/plugin-proposal-async-generator-functions" "^7.8.3"
+ "@babel/plugin-proposal-dynamic-import" "^7.8.3"
+ "@babel/plugin-proposal-json-strings" "^7.8.3"
+ "@babel/plugin-proposal-nullish-coalescing-operator" "^7.8.3"
+ "@babel/plugin-proposal-numeric-separator" "^7.8.3"
+ "@babel/plugin-proposal-object-rest-spread" "^7.9.0"
+ "@babel/plugin-proposal-optional-catch-binding" "^7.8.3"
+ "@babel/plugin-proposal-optional-chaining" "^7.9.0"
+ "@babel/plugin-proposal-unicode-property-regex" "^7.8.3"
+ "@babel/plugin-syntax-async-generators" "^7.8.0"
+ "@babel/plugin-syntax-dynamic-import" "^7.8.0"
+ "@babel/plugin-syntax-json-strings" "^7.8.0"
+ "@babel/plugin-syntax-nullish-coalescing-operator" "^7.8.0"
+ "@babel/plugin-syntax-numeric-separator" "^7.8.0"
+ "@babel/plugin-syntax-object-rest-spread" "^7.8.0"
+ "@babel/plugin-syntax-optional-catch-binding" "^7.8.0"
+ "@babel/plugin-syntax-optional-chaining" "^7.8.0"
+ "@babel/plugin-syntax-top-level-await" "^7.8.3"
+ "@babel/plugin-transform-arrow-functions" "^7.8.3"
+ "@babel/plugin-transform-async-to-generator" "^7.8.3"
+ "@babel/plugin-transform-block-scoped-functions" "^7.8.3"
+ "@babel/plugin-transform-block-scoping" "^7.8.3"
+ "@babel/plugin-transform-classes" "^7.9.0"
+ "@babel/plugin-transform-computed-properties" "^7.8.3"
+ "@babel/plugin-transform-destructuring" "^7.8.3"
+ "@babel/plugin-transform-dotall-regex" "^7.8.3"
+ "@babel/plugin-transform-duplicate-keys" "^7.8.3"
+ "@babel/plugin-transform-exponentiation-operator" "^7.8.3"
+ "@babel/plugin-transform-for-of" "^7.9.0"
+ "@babel/plugin-transform-function-name" "^7.8.3"
+ "@babel/plugin-transform-literals" "^7.8.3"
+ "@babel/plugin-transform-member-expression-literals" "^7.8.3"
+ "@babel/plugin-transform-modules-amd" "^7.9.0"
+ "@babel/plugin-transform-modules-commonjs" "^7.9.0"
+ "@babel/plugin-transform-modules-systemjs" "^7.9.0"
+ "@babel/plugin-transform-modules-umd" "^7.9.0"
+ "@babel/plugin-transform-named-capturing-groups-regex" "^7.8.3"
+ "@babel/plugin-transform-new-target" "^7.8.3"
+ "@babel/plugin-transform-object-super" "^7.8.3"
+ "@babel/plugin-transform-parameters" "^7.8.7"
+ "@babel/plugin-transform-property-literals" "^7.8.3"
+ "@babel/plugin-transform-regenerator" "^7.8.7"
+ "@babel/plugin-transform-reserved-words" "^7.8.3"
+ "@babel/plugin-transform-shorthand-properties" "^7.8.3"
+ "@babel/plugin-transform-spread" "^7.8.3"
+ "@babel/plugin-transform-sticky-regex" "^7.8.3"
+ "@babel/plugin-transform-template-literals" "^7.8.3"
+ "@babel/plugin-transform-typeof-symbol" "^7.8.4"
+ "@babel/plugin-transform-unicode-regex" "^7.8.3"
+ "@babel/preset-modules" "^0.1.3"
+ "@babel/types" "^7.9.0"
+ browserslist "^4.9.1"
+ core-js-compat "^3.6.2"
invariant "^2.2.2"
- js-levenshtein "^1.1.3"
+ levenary "^1.1.1"
semver "^5.5.0"
-"@babel/preset-env@^7.1.6":
- version "7.7.7"
- resolved "https://registry.yarnpkg.com/@babel/preset-env/-/preset-env-7.7.7.tgz#c294167b91e53e7e36d820e943ece8d0c7fe46ac"
- integrity sha512-pCu0hrSSDVI7kCVUOdcMNQEbOPJ52E+LrQ14sN8uL2ALfSqePZQlKrOy+tM4uhEdYlCHi4imr8Zz2cZe9oSdIg==
+"@babel/preset-env@^7.4.5":
+ version "7.9.6"
+ resolved "https://registry.yarnpkg.com/@babel/preset-env/-/preset-env-7.9.6.tgz#df063b276c6455ec6fcfc6e53aacc38da9b0aea6"
+ integrity sha512-0gQJ9RTzO0heXOhzftog+a/WyOuqMrAIugVYxMYf83gh1CQaQDjMtsOpqOwXyDL/5JcWsrCm8l4ju8QC97O7EQ==
dependencies:
- "@babel/helper-module-imports" "^7.7.4"
- "@babel/helper-plugin-utils" "^7.0.0"
- "@babel/plugin-proposal-async-generator-functions" "^7.7.4"
- "@babel/plugin-proposal-dynamic-import" "^7.7.4"
- "@babel/plugin-proposal-json-strings" "^7.7.4"
- "@babel/plugin-proposal-object-rest-spread" "^7.7.7"
- "@babel/plugin-proposal-optional-catch-binding" "^7.7.4"
- "@babel/plugin-proposal-unicode-property-regex" "^7.7.7"
- "@babel/plugin-syntax-async-generators" "^7.7.4"
- "@babel/plugin-syntax-dynamic-import" "^7.7.4"
- "@babel/plugin-syntax-json-strings" "^7.7.4"
- "@babel/plugin-syntax-object-rest-spread" "^7.7.4"
- "@babel/plugin-syntax-optional-catch-binding" "^7.7.4"
- "@babel/plugin-syntax-top-level-await" "^7.7.4"
- "@babel/plugin-transform-arrow-functions" "^7.7.4"
- "@babel/plugin-transform-async-to-generator" "^7.7.4"
- "@babel/plugin-transform-block-scoped-functions" "^7.7.4"
- "@babel/plugin-transform-block-scoping" "^7.7.4"
- "@babel/plugin-transform-classes" "^7.7.4"
- "@babel/plugin-transform-computed-properties" "^7.7.4"
- "@babel/plugin-transform-destructuring" "^7.7.4"
- "@babel/plugin-transform-dotall-regex" "^7.7.7"
- "@babel/plugin-transform-duplicate-keys" "^7.7.4"
- "@babel/plugin-transform-exponentiation-operator" "^7.7.4"
- "@babel/plugin-transform-for-of" "^7.7.4"
- "@babel/plugin-transform-function-name" "^7.7.4"
- "@babel/plugin-transform-literals" "^7.7.4"
- "@babel/plugin-transform-member-expression-literals" "^7.7.4"
- "@babel/plugin-transform-modules-amd" "^7.7.5"
- "@babel/plugin-transform-modules-commonjs" "^7.7.5"
- "@babel/plugin-transform-modules-systemjs" "^7.7.4"
- "@babel/plugin-transform-modules-umd" "^7.7.4"
- "@babel/plugin-transform-named-capturing-groups-regex" "^7.7.4"
- "@babel/plugin-transform-new-target" "^7.7.4"
- "@babel/plugin-transform-object-super" "^7.7.4"
- "@babel/plugin-transform-parameters" "^7.7.7"
- "@babel/plugin-transform-property-literals" "^7.7.4"
- "@babel/plugin-transform-regenerator" "^7.7.5"
- "@babel/plugin-transform-reserved-words" "^7.7.4"
- "@babel/plugin-transform-shorthand-properties" "^7.7.4"
- "@babel/plugin-transform-spread" "^7.7.4"
- "@babel/plugin-transform-sticky-regex" "^7.7.4"
- "@babel/plugin-transform-template-literals" "^7.7.4"
- "@babel/plugin-transform-typeof-symbol" "^7.7.4"
- "@babel/plugin-transform-unicode-regex" "^7.7.4"
- "@babel/types" "^7.7.4"
- browserslist "^4.6.0"
- core-js-compat "^3.6.0"
+ "@babel/compat-data" "^7.9.6"
+ "@babel/helper-compilation-targets" "^7.9.6"
+ "@babel/helper-module-imports" "^7.8.3"
+ "@babel/helper-plugin-utils" "^7.8.3"
+ "@babel/plugin-proposal-async-generator-functions" "^7.8.3"
+ "@babel/plugin-proposal-dynamic-import" "^7.8.3"
+ "@babel/plugin-proposal-json-strings" "^7.8.3"
+ "@babel/plugin-proposal-nullish-coalescing-operator" "^7.8.3"
+ "@babel/plugin-proposal-numeric-separator" "^7.8.3"
+ "@babel/plugin-proposal-object-rest-spread" "^7.9.6"
+ "@babel/plugin-proposal-optional-catch-binding" "^7.8.3"
+ "@babel/plugin-proposal-optional-chaining" "^7.9.0"
+ "@babel/plugin-proposal-unicode-property-regex" "^7.8.3"
+ "@babel/plugin-syntax-async-generators" "^7.8.0"
+ "@babel/plugin-syntax-dynamic-import" "^7.8.0"
+ "@babel/plugin-syntax-json-strings" "^7.8.0"
+ "@babel/plugin-syntax-nullish-coalescing-operator" "^7.8.0"
+ "@babel/plugin-syntax-numeric-separator" "^7.8.0"
+ "@babel/plugin-syntax-object-rest-spread" "^7.8.0"
+ "@babel/plugin-syntax-optional-catch-binding" "^7.8.0"
+ "@babel/plugin-syntax-optional-chaining" "^7.8.0"
+ "@babel/plugin-syntax-top-level-await" "^7.8.3"
+ "@babel/plugin-transform-arrow-functions" "^7.8.3"
+ "@babel/plugin-transform-async-to-generator" "^7.8.3"
+ "@babel/plugin-transform-block-scoped-functions" "^7.8.3"
+ "@babel/plugin-transform-block-scoping" "^7.8.3"
+ "@babel/plugin-transform-classes" "^7.9.5"
+ "@babel/plugin-transform-computed-properties" "^7.8.3"
+ "@babel/plugin-transform-destructuring" "^7.9.5"
+ "@babel/plugin-transform-dotall-regex" "^7.8.3"
+ "@babel/plugin-transform-duplicate-keys" "^7.8.3"
+ "@babel/plugin-transform-exponentiation-operator" "^7.8.3"
+ "@babel/plugin-transform-for-of" "^7.9.0"
+ "@babel/plugin-transform-function-name" "^7.8.3"
+ "@babel/plugin-transform-literals" "^7.8.3"
+ "@babel/plugin-transform-member-expression-literals" "^7.8.3"
+ "@babel/plugin-transform-modules-amd" "^7.9.6"
+ "@babel/plugin-transform-modules-commonjs" "^7.9.6"
+ "@babel/plugin-transform-modules-systemjs" "^7.9.6"
+ "@babel/plugin-transform-modules-umd" "^7.9.0"
+ "@babel/plugin-transform-named-capturing-groups-regex" "^7.8.3"
+ "@babel/plugin-transform-new-target" "^7.8.3"
+ "@babel/plugin-transform-object-super" "^7.8.3"
+ "@babel/plugin-transform-parameters" "^7.9.5"
+ "@babel/plugin-transform-property-literals" "^7.8.3"
+ "@babel/plugin-transform-regenerator" "^7.8.7"
+ "@babel/plugin-transform-reserved-words" "^7.8.3"
+ "@babel/plugin-transform-shorthand-properties" "^7.8.3"
+ "@babel/plugin-transform-spread" "^7.8.3"
+ "@babel/plugin-transform-sticky-regex" "^7.8.3"
+ "@babel/plugin-transform-template-literals" "^7.8.3"
+ "@babel/plugin-transform-typeof-symbol" "^7.8.4"
+ "@babel/plugin-transform-unicode-regex" "^7.8.3"
+ "@babel/preset-modules" "^0.1.3"
+ "@babel/types" "^7.9.6"
+ browserslist "^4.11.1"
+ core-js-compat "^3.6.2"
invariant "^2.2.2"
- js-levenshtein "^1.1.3"
+ levenary "^1.1.1"
semver "^5.5.0"
-"@babel/preset-react@7.0.0":
- version "7.0.0"
- resolved "https://registry.yarnpkg.com/@babel/preset-react/-/preset-react-7.0.0.tgz#e86b4b3d99433c7b3e9e91747e2653958bc6b3c0"
- integrity sha512-oayxyPS4Zj+hF6Et11BwuBkmpgT/zMxyuZgFrMeZID6Hdh3dGlk4sHCAhdBCpuCKW2ppBfl2uCCetlrUIJRY3w==
+"@babel/preset-modules@^0.1.3":
+ version "0.1.3"
+ resolved "https://registry.yarnpkg.com/@babel/preset-modules/-/preset-modules-0.1.3.tgz#13242b53b5ef8c883c3cf7dddd55b36ce80fbc72"
+ integrity sha512-Ra3JXOHBq2xd56xSF7lMKXdjBn3T772Y1Wet3yWnkDly9zHvJki029tAFzvAAK5cf4YV3yoxuP61crYRol6SVg==
dependencies:
"@babel/helper-plugin-utils" "^7.0.0"
- "@babel/plugin-transform-react-display-name" "^7.0.0"
- "@babel/plugin-transform-react-jsx" "^7.0.0"
- "@babel/plugin-transform-react-jsx-self" "^7.0.0"
- "@babel/plugin-transform-react-jsx-source" "^7.0.0"
+ "@babel/plugin-proposal-unicode-property-regex" "^7.4.4"
+ "@babel/plugin-transform-dotall-regex" "^7.4.4"
+ "@babel/types" "^7.4.4"
+ esutils "^2.0.2"
+
+"@babel/preset-react@7.9.1":
+ version "7.9.1"
+ resolved "https://registry.yarnpkg.com/@babel/preset-react/-/preset-react-7.9.1.tgz#b346403c36d58c3bb544148272a0cefd9c28677a"
+ integrity sha512-aJBYF23MPj0RNdp/4bHnAP0NVqqZRr9kl0NAOP4nJCex6OYVio59+dnQzsAWFuogdLyeaKA1hmfUIVZkY5J+TQ==
+ dependencies:
+ "@babel/helper-plugin-utils" "^7.8.3"
+ "@babel/plugin-transform-react-display-name" "^7.8.3"
+ "@babel/plugin-transform-react-jsx" "^7.9.1"
+ "@babel/plugin-transform-react-jsx-development" "^7.9.0"
+ "@babel/plugin-transform-react-jsx-self" "^7.9.0"
+ "@babel/plugin-transform-react-jsx-source" "^7.9.0"
"@babel/preset-react@^7.0.0":
version "7.7.4"
@@ -924,13 +1127,13 @@
"@babel/plugin-transform-react-jsx-self" "^7.7.4"
"@babel/plugin-transform-react-jsx-source" "^7.7.4"
-"@babel/preset-typescript@7.3.3":
- version "7.3.3"
- resolved "https://registry.yarnpkg.com/@babel/preset-typescript/-/preset-typescript-7.3.3.tgz#88669911053fa16b2b276ea2ede2ca603b3f307a"
- integrity sha512-mzMVuIP4lqtn4du2ynEfdO0+RYcslwrZiJHXu4MGaC1ctJiW2fyaeDrtjJGs7R/KebZ1sgowcIoWf4uRpEfKEg==
+"@babel/preset-typescript@7.9.0":
+ version "7.9.0"
+ resolved "https://registry.yarnpkg.com/@babel/preset-typescript/-/preset-typescript-7.9.0.tgz#87705a72b1f0d59df21c179f7c3d2ef4b16ce192"
+ integrity sha512-S4cueFnGrIbvYJgwsVFKdvOmpiL0XGw9MFW9D0vgRys5g36PBhZRL8NX8Gr2akz8XRtzq6HuDXPD/1nniagNUg==
dependencies:
- "@babel/helper-plugin-utils" "^7.0.0"
- "@babel/plugin-transform-typescript" "^7.3.2"
+ "@babel/helper-plugin-utils" "^7.8.3"
+ "@babel/plugin-transform-typescript" "^7.9.0"
"@babel/runtime-corejs3@^7.7.4":
version "7.7.7"
@@ -940,20 +1143,35 @@
core-js-pure "^3.0.0"
regenerator-runtime "^0.13.2"
-"@babel/runtime@7.4.3":
- version "7.4.3"
- resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.4.3.tgz#79888e452034223ad9609187a0ad1fe0d2ad4bdc"
- integrity sha512-9lsJwJLxDh/T3Q3SZszfWOTkk3pHbkmH+3KY+zwIDmsNlxsumuhS2TH3NIpktU4kNvfzy+k3eLT7aTJSPTo0OA==
+"@babel/runtime-corejs3@^7.8.3":
+ version "7.9.6"
+ resolved "https://registry.yarnpkg.com/@babel/runtime-corejs3/-/runtime-corejs3-7.9.6.tgz#67aded13fffbbc2cb93247388cf84d77a4be9a71"
+ integrity sha512-6toWAfaALQjt3KMZQc6fABqZwUDDuWzz+cAfPhqyEnzxvdWOAkjwPNxgF8xlmo7OWLsSjaKjsskpKHRLaMArOA==
dependencies:
- regenerator-runtime "^0.13.2"
+ core-js-pure "^3.0.0"
+ regenerator-runtime "^0.13.4"
-"@babel/runtime@^7.0.0", "@babel/runtime@^7.1.2", "@babel/runtime@^7.3.4", "@babel/runtime@^7.4.0", "@babel/runtime@^7.4.2", "@babel/runtime@^7.7.4":
+"@babel/runtime@7.9.0":
+ version "7.9.0"
+ resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.9.0.tgz#337eda67401f5b066a6f205a3113d4ac18ba495b"
+ integrity sha512-cTIudHnzuWLS56ik4DnRnqqNf8MkdUzV4iFFI1h7Jo9xvrpQROYaAnaSd2mHLQAzzZAPfATynX5ord6YlNYNMA==
+ dependencies:
+ regenerator-runtime "^0.13.4"
+
+"@babel/runtime@^7.0.0", "@babel/runtime@^7.1.2", "@babel/runtime@^7.3.4", "@babel/runtime@^7.4.0", "@babel/runtime@^7.7.4":
version "7.7.7"
resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.7.7.tgz#194769ca8d6d7790ec23605af9ee3e42a0aa79cf"
integrity sha512-uCnC2JEVAu8AKB5do1WRIsvrdJ0flYx/A/9f/6chdacnEZ7LmavjdsDXr5ksYBegxtuTPR5Va9/+13QF/kFkCA==
dependencies:
regenerator-runtime "^0.13.2"
+"@babel/runtime@^7.4.5", "@babel/runtime@^7.7.2", "@babel/runtime@^7.8.4", "@babel/runtime@^7.9.2":
+ version "7.9.6"
+ resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.9.6.tgz#a9102eb5cadedf3f31d08a9ecf294af7827ea29f"
+ integrity sha512-64AF1xY3OAkFHqOb9s4jpgk1Mm5vDZ4L3acHvAml+53nO1XbXLuDodsVpO4OIUsmemlUHMxNdYMNJmsvOwLrvQ==
+ dependencies:
+ regenerator-runtime "^0.13.4"
+
"@babel/template@^7.4.0", "@babel/template@^7.7.4":
version "7.7.4"
resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.7.4.tgz#428a7d9eecffe27deac0a98e23bf8e3675d2a77b"
@@ -963,7 +1181,16 @@
"@babel/parser" "^7.7.4"
"@babel/types" "^7.7.4"
-"@babel/traverse@^7.0.0", "@babel/traverse@^7.1.0", "@babel/traverse@^7.4.3", "@babel/traverse@^7.7.4":
+"@babel/template@^7.8.3", "@babel/template@^7.8.6":
+ version "7.8.6"
+ resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.8.6.tgz#86b22af15f828dfb086474f964dcc3e39c43ce2b"
+ integrity sha512-zbMsPMy/v0PWFZEhQJ66bqjhH+z0JgMoBWuikXybgG3Gkd/3t5oQ1Rw2WQhnSrsOmsKXnZOx15tkC4qON/+JPg==
+ dependencies:
+ "@babel/code-frame" "^7.8.3"
+ "@babel/parser" "^7.8.6"
+ "@babel/types" "^7.8.6"
+
+"@babel/traverse@^7.1.0", "@babel/traverse@^7.4.3", "@babel/traverse@^7.7.4":
version "7.7.4"
resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.7.4.tgz#9c1e7c60fb679fe4fcfaa42500833333c2058558"
integrity sha512-P1L58hQyupn8+ezVA2z5KBm4/Zr4lCC8dwKCMYzsa5jFMDMQAzaBNy9W5VjB+KAmBjb40U7a/H6ao+Xo+9saIw==
@@ -978,6 +1205,21 @@
globals "^11.1.0"
lodash "^4.17.13"
+"@babel/traverse@^7.7.0", "@babel/traverse@^7.8.3", "@babel/traverse@^7.9.0", "@babel/traverse@^7.9.6":
+ version "7.9.6"
+ resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.9.6.tgz#5540d7577697bf619cc57b92aa0f1c231a94f442"
+ integrity sha512-b3rAHSjbxy6VEAvlxM8OV/0X4XrG72zoxme6q1MOoe2vd0bEc+TwayhuC1+Dfgqh1QEG+pj7atQqvUprHIccsg==
+ dependencies:
+ "@babel/code-frame" "^7.8.3"
+ "@babel/generator" "^7.9.6"
+ "@babel/helper-function-name" "^7.9.5"
+ "@babel/helper-split-export-declaration" "^7.8.3"
+ "@babel/parser" "^7.9.6"
+ "@babel/types" "^7.9.6"
+ debug "^4.1.0"
+ globals "^11.1.0"
+ lodash "^4.17.13"
+
"@babel/types@^7.0.0", "@babel/types@^7.3.0", "@babel/types@^7.4.0", "@babel/types@^7.4.4", "@babel/types@^7.7.4":
version "7.7.4"
resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.7.4.tgz#516570d539e44ddf308c07569c258ff94fde9193"
@@ -987,6 +1229,15 @@
lodash "^4.17.13"
to-fast-properties "^2.0.0"
+"@babel/types@^7.7.0", "@babel/types@^7.8.3", "@babel/types@^7.8.6", "@babel/types@^7.9.0", "@babel/types@^7.9.5", "@babel/types@^7.9.6":
+ version "7.9.6"
+ resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.9.6.tgz#2c5502b427251e9de1bd2dff95add646d95cc9f7"
+ integrity sha512-qxXzvBO//jO9ZnoasKF1uJzHd2+M6Q2ZPIVfnFps8JJvXy0ZBbwbNOmE6SGIY5XOY6d1Bo5lb9d9RJ8nv3WSeA==
+ dependencies:
+ "@babel/helper-validator-identifier" "^7.9.5"
+ lodash "^4.17.13"
+ to-fast-properties "^2.0.0"
+
"@cnakazawa/watch@^1.0.3":
version "1.0.3"
resolved "https://registry.yarnpkg.com/@cnakazawa/watch/-/watch-1.0.3.tgz#099139eaec7ebf07a27c1786a3ff64f39464d2ef"
@@ -1000,10 +1251,10 @@
resolved "https://registry.yarnpkg.com/@csstools/convert-colors/-/convert-colors-1.4.0.tgz#ad495dc41b12e75d588c6db8b9834f08fa131eb7"
integrity sha512-5a6wqoJV/xEdbRNKVo6I4hO3VjyDq//8q2f9I6PBAvMesJHFauXDorcNCsr9RzvsZnaWi5NYCcfyqP1QeFHFbw==
-"@csstools/normalize.css@^9.0.1":
- version "9.0.1"
- resolved "https://registry.yarnpkg.com/@csstools/normalize.css/-/normalize.css-9.0.1.tgz#c27b391d8457d1e893f1eddeaf5e5412d12ffbb5"
- integrity sha512-6It2EVfGskxZCQhuykrfnALg7oVeiI6KclWSmGDqB0AiInVrTGB9Jp9i4/Ad21u9Jde/voVQz6eFX/eSg/UsPA==
+"@csstools/normalize.css@^10.1.0":
+ version "10.1.0"
+ resolved "https://registry.yarnpkg.com/@csstools/normalize.css/-/normalize.css-10.1.0.tgz#f0950bba18819512d42f7197e56c518aa491cf18"
+ integrity sha512-ij4wRiunFfaJxjB0BdrYHIH8FxBJpOwNPhhAcunlmPdXudL1WQV1qoP9un6JsEBAgQH+7UXyyjh0g7jTxXK6tg==
"@hapi/address@2.x.x":
version "2.1.4"
@@ -1080,7 +1331,7 @@
slash "^2.0.0"
strip-ansi "^5.0.0"
-"@jest/environment@^24.9.0":
+"@jest/environment@^24.3.0", "@jest/environment@^24.9.0":
version "24.9.0"
resolved "https://registry.yarnpkg.com/@jest/environment/-/environment-24.9.0.tgz#21e3afa2d65c0586cbd6cbefe208bafade44ab18"
integrity sha512-5A1QluTPhvdIPFYnO3sZC3smkNeXPVELz7ikPbhUj0bQjB07EoE9qtLrem14ZUYWdVayYbsjVwIiL4WBIMV4aQ==
@@ -1090,7 +1341,7 @@
"@jest/types" "^24.9.0"
jest-mock "^24.9.0"
-"@jest/fake-timers@^24.9.0":
+"@jest/fake-timers@^24.3.0", "@jest/fake-timers@^24.9.0":
version "24.9.0"
resolved "https://registry.yarnpkg.com/@jest/fake-timers/-/fake-timers-24.9.0.tgz#ba3e6bf0eecd09a636049896434d306636540c93"
integrity sha512-eWQcNa2YSwzXWIMC5KufBh3oWRIijrQFROsIqt6v/NS9Io/gknw1jsAC9c+ih/RQX4A3O7SeWAhQeN0goKhT9A==
@@ -1154,7 +1405,7 @@
jest-runner "^24.9.0"
jest-runtime "^24.9.0"
-"@jest/transform@^24.7.1", "@jest/transform@^24.9.0":
+"@jest/transform@^24.9.0":
version "24.9.0"
resolved "https://registry.yarnpkg.com/@jest/transform/-/transform-24.9.0.tgz#4ae2768b296553fadab09e9ec119543c90b16c56"
integrity sha512-TcQUmyNRxV94S0QpMOnZl0++6RMiqpbH/ZMccFB/amku6Uwvyb1cjYX7xkp5nGNkbX4QPH/FcB6q1HBTHynLmQ==
@@ -1176,7 +1427,7 @@
source-map "^0.6.1"
write-file-atomic "2.4.1"
-"@jest/types@^24.7.0", "@jest/types@^24.9.0":
+"@jest/types@^24.3.0", "@jest/types@^24.9.0":
version "24.9.0"
resolved "https://registry.yarnpkg.com/@jest/types/-/types-24.9.0.tgz#63cb26cb7500d069e5a389441a7c6ab5e909fc59"
integrity sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==
@@ -1257,7 +1508,7 @@
"@svgr/babel-plugin-transform-react-native-svg" "^4.2.0"
"@svgr/babel-plugin-transform-svg-component" "^4.2.0"
-"@svgr/core@^4.1.0":
+"@svgr/core@^4.3.3":
version "4.3.3"
resolved "https://registry.yarnpkg.com/@svgr/core/-/core-4.3.3.tgz#b37b89d5b757dc66e8c74156d00c368338d24293"
integrity sha512-qNuGF1QON1626UCaZamWt5yedpgOytvLj5BQZe2j1k1B8DUG4OyugZyfEwBeXozCUwhLEpsrgPrE+eCu4fY17w==
@@ -1273,7 +1524,7 @@
dependencies:
"@babel/types" "^7.4.4"
-"@svgr/plugin-jsx@^4.1.0", "@svgr/plugin-jsx@^4.3.3":
+"@svgr/plugin-jsx@^4.3.3":
version "4.3.3"
resolved "https://registry.yarnpkg.com/@svgr/plugin-jsx/-/plugin-jsx-4.3.3.tgz#e2ba913dbdfbe85252a34db101abc7ebd50992fa"
integrity sha512-cLOCSpNWQnDB1/v+SUENHH7a0XY09bfuMKdq9+gYvtuwzC2rU4I0wKGFEp1i24holdQdwodCtDQdFtJiTCWc+w==
@@ -1283,7 +1534,7 @@
"@svgr/hast-util-to-babel-ast" "^4.3.2"
svg-parser "^2.0.0"
-"@svgr/plugin-svgo@^4.0.3":
+"@svgr/plugin-svgo@^4.3.1":
version "4.3.1"
resolved "https://registry.yarnpkg.com/@svgr/plugin-svgo/-/plugin-svgo-4.3.1.tgz#daac0a3d872e3f55935c6588dd370336865e9e32"
integrity sha512-PrMtEDUWjX3Ea65JsVCwTIXuSqa3CG9px+DluF1/eo9mlDrgrtFE7NE/DjdhjJgSM9wenlVBzkzneSIUgfUI/w==
@@ -1292,19 +1543,19 @@
merge-deep "^3.0.2"
svgo "^1.2.2"
-"@svgr/webpack@4.1.0":
- version "4.1.0"
- resolved "https://registry.yarnpkg.com/@svgr/webpack/-/webpack-4.1.0.tgz#20c88f32f731c7b1d4711045b2b993887d731c28"
- integrity sha512-d09ehQWqLMywP/PT/5JvXwPskPK9QCXUjiSkAHehreB381qExXf5JFCBWhfEyNonRbkIneCeYM99w+Ud48YIQQ==
+"@svgr/webpack@4.3.3":
+ version "4.3.3"
+ resolved "https://registry.yarnpkg.com/@svgr/webpack/-/webpack-4.3.3.tgz#13cc2423bf3dff2d494f16b17eb7eacb86895017"
+ integrity sha512-bjnWolZ6KVsHhgyCoYRFmbd26p8XVbulCzSG53BDQqAr+JOAderYK7CuYrB3bDjHJuF6LJ7Wrr42+goLRV9qIg==
dependencies:
- "@babel/core" "^7.1.6"
+ "@babel/core" "^7.4.5"
"@babel/plugin-transform-react-constant-elements" "^7.0.0"
- "@babel/preset-env" "^7.1.6"
+ "@babel/preset-env" "^7.4.5"
"@babel/preset-react" "^7.0.0"
- "@svgr/core" "^4.1.0"
- "@svgr/plugin-jsx" "^4.1.0"
- "@svgr/plugin-svgo" "^4.0.3"
- loader-utils "^1.1.0"
+ "@svgr/core" "^4.3.3"
+ "@svgr/plugin-jsx" "^4.3.3"
+ "@svgr/plugin-svgo" "^4.3.1"
+ loader-utils "^1.2.3"
"@szmarczak/http-timer@^1.1.2":
version "1.1.2"
@@ -1346,6 +1597,35 @@
dependencies:
"@babel/types" "^7.3.0"
+"@types/classnames@^2.2.10":
+ version "2.2.10"
+ resolved "https://registry.yarnpkg.com/@types/classnames/-/classnames-2.2.10.tgz#cc658ca319b6355399efc1f5b9e818f1a24bf999"
+ integrity sha512-1UzDldn9GfYYEsWWnn/P4wkTlkZDH7lDb0wBMGbtIQc9zXEQq7FlKBdZUn6OBqD8sKZZ2RQO2mAjGpXiDGoRmQ==
+
+"@types/color-name@^1.1.1":
+ version "1.1.1"
+ resolved "https://registry.yarnpkg.com/@types/color-name/-/color-name-1.1.1.tgz#1c1261bbeaa10a8055bbc5d8ab84b7b2afc846a0"
+ integrity sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==
+
+"@types/eslint-visitor-keys@^1.0.0":
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/@types/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz#1ee30d79544ca84d68d4b3cdb0af4f205663dd2d"
+ integrity sha512-OCutwjDZ4aFS6PB1UZ988C4YgwlBHJd6wCeQqaLdmadZ/7e+w79+hbMUFC1QXDNCmdyoRfAFdm0RypzwR+Qpag==
+
+"@types/events@*":
+ version "3.0.0"
+ resolved "https://registry.yarnpkg.com/@types/events/-/events-3.0.0.tgz#2862f3f58a9a7f7c3e78d79f130dd4d71c25c2a7"
+ integrity sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g==
+
+"@types/glob@^7.1.1":
+ version "7.1.1"
+ resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.1.1.tgz#aa59a1c6e3fbc421e07ccd31a944c30eba521575"
+ integrity sha512-1Bh06cbWJUHMC97acuD6UMG29nMt0Aqz1vF3guLfG+kHHJhy3AyohZFFxYk2f7Q1SQIrNwvncxAE0N/9s70F2w==
+ dependencies:
+ "@types/events" "*"
+ "@types/minimatch" "*"
+ "@types/node" "*"
+
"@types/history@*":
version "4.7.3"
resolved "https://registry.yarnpkg.com/@types/history/-/history-4.7.3.tgz#856c99cdc1551d22c22b18b5402719affec9839a"
@@ -1385,11 +1665,36 @@
dependencies:
"@types/jest-diff" "*"
+"@types/json-schema@^7.0.3":
+ version "7.0.4"
+ resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.4.tgz#38fd73ddfd9b55abb1e1b2ed578cb55bd7b7d339"
+ integrity sha512-8+KAKzEvSUdeo+kmqnKrqgeE+LcA0tjYWFY7RPProVYwnqDjukzO+3b6dLD56rYX5TdWejnEOLJYOIeh4CXKuA==
+
+"@types/minimatch@*":
+ version "3.0.3"
+ resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d"
+ integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==
+
+"@types/node@*":
+ version "13.13.5"
+ resolved "https://registry.yarnpkg.com/@types/node/-/node-13.13.5.tgz#96ec3b0afafd64a4ccea9107b75bf8489f0e5765"
+ integrity sha512-3ySmiBYJPqgjiHA7oEaIo2Rzz0HrOZ7yrNO5HWyaE5q0lQ3BppDZ3N53Miz8bw2I7gh1/zir2MGVZBvpb1zq9g==
+
"@types/node@11.13.9":
version "11.13.9"
resolved "https://registry.yarnpkg.com/@types/node/-/node-11.13.9.tgz#f80697caca7f7fb2526527a5c5a2743487f05ccc"
integrity sha512-NJ4yuEVw5podZbINp3tEqUIImMSAEHaCXRiWCf3KC32l6hIKf0iPJEh2uZdT0fELfRYk310yLmMXqy2leZQUbg==
+"@types/normalize-package-data@^2.4.0":
+ version "2.4.0"
+ resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz#e486d0d97396d79beedd0a6e33f4534ff6b4973e"
+ integrity sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA==
+
+"@types/parse-json@^4.0.0":
+ version "4.0.0"
+ resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0"
+ integrity sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==
+
"@types/prop-types@*":
version "15.7.3"
resolved "https://registry.yarnpkg.com/@types/prop-types/-/prop-types-15.7.3.tgz#2ab0d5da2e5815f94b0b9d4b95d1e5f243ab2ca7"
@@ -1464,32 +1769,48 @@
dependencies:
"@types/yargs-parser" "*"
-"@typescript-eslint/eslint-plugin@1.6.0":
- version "1.6.0"
- resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-1.6.0.tgz#a5ff3128c692393fb16efa403ec7c8a5593dab0f"
- integrity sha512-U224c29E2lo861TQZs6GSmyC0OYeRNg6bE9UVIiFBxN2MlA0nq2dCrgIVyyRbC05UOcrgf2Wk/CF2gGOPQKUSQ==
+"@typescript-eslint/eslint-plugin@^2.10.0", "@typescript-eslint/eslint-plugin@^2.29.0", "@typescript-eslint/eslint-plugin@^2.31.0":
+ version "2.31.0"
+ resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-2.31.0.tgz#942c921fec5e200b79593c71fafb1e3f57aa2e36"
+ integrity sha512-iIC0Pb8qDaoit+m80Ln/aaeu9zKQdOLF4SHcGLarSeY1gurW6aU4JsOPMjKQwXlw70MvWKZQc6S2NamA8SJ/gg==
dependencies:
- "@typescript-eslint/parser" "1.6.0"
- "@typescript-eslint/typescript-estree" "1.6.0"
- requireindex "^1.2.0"
- tsutils "^3.7.0"
+ "@typescript-eslint/experimental-utils" "2.31.0"
+ functional-red-black-tree "^1.0.1"
+ regexpp "^3.0.0"
+ tsutils "^3.17.1"
-"@typescript-eslint/parser@1.6.0":
- version "1.6.0"
- resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-1.6.0.tgz#f01189c8b90848e3b8e45a6cdad27870529d1804"
- integrity sha512-VB9xmSbfafI+/kI4gUK3PfrkGmrJQfh0N4EScT1gZXSZyUxpsBirPL99EWZg9MmPG0pzq/gMtgkk7/rAHj4aQw==
+"@typescript-eslint/experimental-utils@2.31.0":
+ version "2.31.0"
+ resolved "https://registry.yarnpkg.com/@typescript-eslint/experimental-utils/-/experimental-utils-2.31.0.tgz#a9ec514bf7fd5e5e82bc10dcb6a86d58baae9508"
+ integrity sha512-MI6IWkutLYQYTQgZ48IVnRXmLR/0Q6oAyJgiOror74arUMh7EWjJkADfirZhRsUMHeLJ85U2iySDwHTSnNi9vA==
dependencies:
- "@typescript-eslint/typescript-estree" "1.6.0"
- eslint-scope "^4.0.0"
- eslint-visitor-keys "^1.0.0"
+ "@types/json-schema" "^7.0.3"
+ "@typescript-eslint/typescript-estree" "2.31.0"
+ eslint-scope "^5.0.0"
+ eslint-utils "^2.0.0"
-"@typescript-eslint/typescript-estree@1.6.0":
- version "1.6.0"
- resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-1.6.0.tgz#6cf43a07fee08b8eb52e4513b428c8cdc9751ef0"
- integrity sha512-A4CanUwfaG4oXobD5y7EXbsOHjCwn8tj1RDd820etpPAjH+Icjc2K9e/DQM1Hac5zH2BSy+u6bjvvF2wwREvYA==
+"@typescript-eslint/parser@^2.10.0", "@typescript-eslint/parser@^2.29.0", "@typescript-eslint/parser@^2.31.0":
+ version "2.31.0"
+ resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-2.31.0.tgz#beddd4e8efe64995108b229b2862cd5752d40d6f"
+ integrity sha512-uph+w6xUOlyV2DLSC6o+fBDzZ5i7+3/TxAsH4h3eC64tlga57oMb96vVlXoMwjR/nN+xyWlsnxtbDkB46M2EPQ==
dependencies:
- lodash.unescape "4.0.1"
- semver "5.5.0"
+ "@types/eslint-visitor-keys" "^1.0.0"
+ "@typescript-eslint/experimental-utils" "2.31.0"
+ "@typescript-eslint/typescript-estree" "2.31.0"
+ eslint-visitor-keys "^1.1.0"
+
+"@typescript-eslint/typescript-estree@2.31.0":
+ version "2.31.0"
+ resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-2.31.0.tgz#ac536c2d46672aa1f27ba0ec2140d53670635cfd"
+ integrity sha512-vxW149bXFXXuBrAak0eKHOzbcu9cvi6iNcJDzEtOkRwGHxJG15chiAQAwhLOsk+86p9GTr/TziYvw+H9kMaIgA==
+ dependencies:
+ debug "^4.1.1"
+ eslint-visitor-keys "^1.1.0"
+ glob "^7.1.6"
+ is-glob "^4.0.1"
+ lodash "^4.17.15"
+ semver "^6.3.0"
+ tsutils "^3.17.1"
"@webassemblyjs/ast@1.8.5":
version "1.8.5"
@@ -1660,11 +1981,6 @@
mime-types "~2.1.24"
negotiator "0.6.2"
-acorn-dynamic-import@^4.0.0:
- version "4.0.0"
- resolved "https://registry.yarnpkg.com/acorn-dynamic-import/-/acorn-dynamic-import-4.0.0.tgz#482210140582a36b83c3e342e1cfebcaa9240948"
- integrity sha512-d3OEjQV4ROpoflsnUA8HozoIR504TFxNivYEUi6uwz0IYhBkTDXGuWlNdMtybRt3nqVx/L6XqMt0FxkXuWKZhw==
-
acorn-globals@^4.1.0, acorn-globals@^4.3.0:
version "4.3.4"
resolved "https://registry.yarnpkg.com/acorn-globals/-/acorn-globals-4.3.4.tgz#9fa1926addc11c97308c4e66d7add0d40c3272e7"
@@ -1673,10 +1989,10 @@
acorn "^6.0.1"
acorn-walk "^6.0.1"
-acorn-jsx@^5.0.0:
- version "5.1.0"
- resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.1.0.tgz#294adb71b57398b0680015f0a38c563ee1db5384"
- integrity sha512-tMUqwBWfLFbJbizRmEcWSLw6HnFzfdJs2sOJEOwwtVPMoH/0Ay+E703oZz78VSXZiiDcZrQ5XKjPIUQixhmgVw==
+acorn-jsx@^5.2.0:
+ version "5.2.0"
+ resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.2.0.tgz#4c66069173d6fdd68ed85239fc256226182b2ebe"
+ integrity sha512-HiUX/+K2YpkpJ+SzBffkM/AQ2YE03S0U1kjTLVpoJdhZMOWy8qvXVN9JdLqv2QsaQ6MPYQIuNmwD8zOiYUofLQ==
acorn-walk@^6.0.1:
version "6.2.0"
@@ -1688,11 +2004,21 @@
resolved "https://registry.yarnpkg.com/acorn/-/acorn-5.7.3.tgz#67aa231bf8812974b85235a96771eb6bd07ea279"
integrity sha512-T/zvzYRfbVojPWahDsE5evJdHb3oJoQfFbsrKM7w5Zcs++Tr257tia3BmMP8XYVjp1S9RZXQMh7gao96BlqZOw==
-acorn@^6.0.1, acorn@^6.0.4, acorn@^6.0.5, acorn@^6.0.7:
+acorn@^6.0.1, acorn@^6.0.4:
version "6.4.0"
resolved "https://registry.yarnpkg.com/acorn/-/acorn-6.4.0.tgz#b659d2ffbafa24baf5db1cdbb2c94a983ecd2784"
integrity sha512-gac8OEcQ2Li1dxIEWGZzsp2BitJxwkwcOm0zHAJLcPJaVvm58FRnk6RkuLRpU1EujipU2ZFODv2P9DLMfnV8mw==
+acorn@^6.2.1:
+ version "6.4.1"
+ resolved "https://registry.yarnpkg.com/acorn/-/acorn-6.4.1.tgz#531e58ba3f51b9dacb9a6646ca4debf5b14ca474"
+ integrity sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==
+
+acorn@^7.1.1:
+ version "7.1.1"
+ resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.1.1.tgz#e35668de0b402f359de515c5482a1ab9f89a69bf"
+ integrity sha512-add7dgA5ppRPxCFJoAGfMDi7PIBXq1RtGo7BhbLaxwrXPOmw8gq48Y9ozT01hUKy9byMjlR20EJhu5zlkErEkg==
+
add-dom-event-listener@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/add-dom-event-listener/-/add-dom-event-listener-1.1.0.tgz#6a92db3a0dd0abc254e095c0f1dc14acbbaae310"
@@ -1705,17 +2031,36 @@
resolved "https://registry.yarnpkg.com/address/-/address-1.1.2.tgz#bf1116c9c758c51b7a933d296b72c221ed9428b6"
integrity sha512-aT6camzM4xEA54YVJYSqxz1kv4IHnQZRtThJJHhUMRExaU5spC7jX5ugSwTaTgJliIgs4VhZOk7htClvQ/LmRA==
+adjust-sourcemap-loader@2.0.0:
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/adjust-sourcemap-loader/-/adjust-sourcemap-loader-2.0.0.tgz#6471143af75ec02334b219f54bc7970c52fb29a4"
+ integrity sha512-4hFsTsn58+YjrU9qKzML2JSSDqKvN8mUGQ0nNIrfPi8hmIONT4L3uUaT6MKdMsZ9AjsU6D2xDkZxCkbQPxChrA==
+ dependencies:
+ assert "1.4.1"
+ camelcase "5.0.0"
+ loader-utils "1.2.3"
+ object-path "0.11.4"
+ regex-parser "2.2.10"
+
+aggregate-error@^3.0.0:
+ version "3.0.1"
+ resolved "https://registry.yarnpkg.com/aggregate-error/-/aggregate-error-3.0.1.tgz#db2fe7246e536f40d9b5442a39e117d7dd6a24e0"
+ integrity sha512-quoaXsZ9/BLNae5yiNoUz+Nhkwz83GhWwtYFglcjEQB2NDHCIpApbqXxIFnm4Pq/Nvhrsq5sYJFyohrrxnTGAA==
+ dependencies:
+ clean-stack "^2.0.0"
+ indent-string "^4.0.0"
+
ajv-errors@^1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/ajv-errors/-/ajv-errors-1.0.1.tgz#f35986aceb91afadec4102fbd85014950cefa64d"
integrity sha512-DCRfO/4nQ+89p/RK43i8Ezd41EqdGIU4ld7nGF8OQ14oc/we5rEntLCUa7+jrn3nn83BosfwZA0wb4pon2o8iQ==
-ajv-keywords@^3.1.0:
+ajv-keywords@^3.1.0, ajv-keywords@^3.4.1:
version "3.4.1"
resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-3.4.1.tgz#ef916e271c64ac12171fd8384eaae6b2345854da"
integrity sha512-RO1ibKvd27e6FEShVFfPALuHI3WjSVNeK5FIsmme/LYRNxjKuNj+Dt7bucLa6NdSv3JcVTyMlm9kGR84z1XpaQ==
-ajv@^6.1.0, ajv@^6.10.2, ajv@^6.5.5, ajv@^6.9.1:
+ajv@^6.1.0, ajv@^6.10.2, ajv@^6.5.5:
version "6.10.2"
resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.10.2.tgz#d3cea04d6b017b2894ad69040fec8b623eb4bd52"
integrity sha512-TXtUUEYHuaTEbLZWIKUr5pmBuhDLy+8KYtPYdcV8qC+pOZL+NKqYwvWSRrVXHn+ZmRRAu8vJTAznH7Oag6RVRw==
@@ -1725,6 +2070,16 @@
json-schema-traverse "^0.4.1"
uri-js "^4.2.2"
+ajv@^6.10.0, ajv@^6.12.0:
+ version "6.12.2"
+ resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.2.tgz#c629c5eced17baf314437918d2da88c99d5958cd"
+ integrity sha512-k+V+hzjm5q/Mr8ef/1Y9goCmlsK4I6Sm74teeyGvFk1XrOsbsKLjEdrvny42CZ+a8sXbk8KWpY/bDwS+FLL2UQ==
+ dependencies:
+ fast-deep-equal "^3.1.1"
+ fast-json-stable-stringify "^2.0.0"
+ json-schema-traverse "^0.4.1"
+ uri-js "^4.2.2"
+
alphanum-sort@^1.0.0:
version "1.0.2"
resolved "https://registry.yarnpkg.com/alphanum-sort/-/alphanum-sort-1.0.2.tgz#97a1119649b211ad33691d9f9f486a8ec9fbe0a3"
@@ -1742,11 +2097,18 @@
resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-3.2.4.tgz#e3a3da4bfbae6c86a9c285625de124a234026fbf"
integrity sha512-hHUXGagefjN2iRrID63xckIvotOXOojhQKWIPUZ4mNUZ9nLZW+7FMNoE1lOkEhNWYsx/7ysGIuJYCiMAA9FnrA==
-ansi-escapes@^3.0.0, ansi-escapes@^3.2.0:
+ansi-escapes@^3.0.0:
version "3.2.0"
resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-3.2.0.tgz#8780b98ff9dbf5638152d1f1fe5c1d7b4442976b"
integrity sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ==
+ansi-escapes@^4.2.1:
+ version "4.3.1"
+ resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-4.3.1.tgz#a5c47cc43181f1f38ffd7076837700d395522a61"
+ integrity sha512-JWF7ocqNrp8u9oqpgV+wH5ftbt+cfvv+PTjOvKLT3AdYly/LmORARfEVT1iyjwN+4MqE5UmVKoAdIBqeoCHgLA==
+ dependencies:
+ type-fest "^0.11.0"
+
ansi-html@0.0.7:
version "0.0.7"
resolved "https://registry.yarnpkg.com/ansi-html/-/ansi-html-0.0.7.tgz#813584021962a9e9e6fd039f940d12f56ca7859e"
@@ -1767,6 +2129,11 @@
resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-4.1.0.tgz#8b9f8f08cf1acb843756a839ca8c7e3168c51997"
integrity sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==
+ansi-regex@^5.0.0:
+ version "5.0.0"
+ resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75"
+ integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==
+
ansi-styles@^2.2.1:
version "2.2.1"
resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe"
@@ -1779,6 +2146,14 @@
dependencies:
color-convert "^1.9.0"
+ansi-styles@^4.1.0:
+ version "4.2.1"
+ resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.2.1.tgz#90ae75c424d008d2624c5bf29ead3177ebfcf359"
+ integrity sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==
+ dependencies:
+ "@types/color-name" "^1.1.1"
+ color-convert "^2.0.1"
+
antd@^3.26.9:
version "3.26.11"
resolved "https://registry.yarnpkg.com/antd/-/antd-3.26.11.tgz#28d17b8c664251b879e25ad22b3102b1d04c5a10"
@@ -1847,6 +2222,14 @@
micromatch "^3.1.4"
normalize-path "^2.1.1"
+anymatch@~3.1.1:
+ version "3.1.1"
+ resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.1.tgz#c55ecf02185e2469259399310c173ce31233b142"
+ integrity sha512-mM8522psRCqzV+6LhomX5wgp25YVibjh8Wj23I5RPkPppSVSjyKD2A2mBJmWGa+KN7f2D6LNh9jkBCeyLktzjg==
+ dependencies:
+ normalize-path "^3.0.0"
+ picomatch "^2.0.4"
+
aproba@^1.1.1:
version "1.2.0"
resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a"
@@ -1867,6 +2250,11 @@
ast-types-flow "0.0.7"
commander "^2.11.0"
+arity-n@^1.0.4:
+ version "1.0.4"
+ resolved "https://registry.yarnpkg.com/arity-n/-/arity-n-1.0.4.tgz#d9e76b11733e08569c0847ae7b39b2860b30b745"
+ integrity sha1-2edrEXM+CFacCEeuezmyhgswt0U=
+
arr-diff@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/arr-diff/-/arr-diff-4.0.0.tgz#d6461074febfec71e7e15235761a329a5dc7c520"
@@ -1887,6 +2275,16 @@
resolved "https://registry.yarnpkg.com/array-equal/-/array-equal-1.0.0.tgz#8c2a5ef2472fd9ea742b04c77a75093ba2757c93"
integrity sha1-jCpe8kcv2ep0KwTHenUJO6J1fJM=
+array-find-index@^1.0.1:
+ version "1.0.2"
+ resolved "https://registry.yarnpkg.com/array-find-index/-/array-find-index-1.0.2.tgz#df010aa1287e164bbda6f9723b0a96a1ec4187a1"
+ integrity sha1-3wEKoSh+Fku9pvlyOwqWoexBh6E=
+
+array-find@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/array-find/-/array-find-1.0.0.tgz#6c8e286d11ed768327f8e62ecee87353ca3e78b8"
+ integrity sha1-bI4obRHtdoMn+OYuzuhzU8o+eLg=
+
array-flatten@1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2"
@@ -1897,7 +2295,7 @@
resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-2.1.2.tgz#24ef80a28c1a893617e2149b0c6d0d788293b099"
integrity sha512-hNfzcOV8W4NdualtqBFPyVO+54DSJuZGY9qT4pRroB6S9e3iiido2ISIC5h9R2sPJ8H3FHCIiEnsv1lPXO3KtQ==
-array-includes@^3.0.3:
+array-includes@^3.0.3, array-includes@^3.1.1:
version "3.1.1"
resolved "https://registry.yarnpkg.com/array-includes/-/array-includes-3.1.1.tgz#cdd67e6852bdf9c1215460786732255ed2459348"
integrity sha512-c2VXaCHl7zPsvpkFsw4nxvFie4fh1ur9bpcgsVkIjqn0H/Xwdg+7fv3n2r/isyS8EBj5b06M9kHyZuIr4El6WQ==
@@ -1911,7 +2309,7 @@
resolved "https://registry.yarnpkg.com/array-tree-filter/-/array-tree-filter-2.1.0.tgz#873ac00fec83749f255ac8dd083814b4f6329190"
integrity sha512-4ROwICNlNw/Hqa9v+rk5h22KjmzB1JGTMVKP2AKJBOCgb0yL0ASf0+YvCcLNNwquOHNX48jkeZIJ3a+oOQqKcw==
-array-union@^1.0.1:
+array-union@^1.0.1, array-union@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/array-union/-/array-union-1.0.2.tgz#9a34410e4f4e3da23dea375be5be70f24778ec39"
integrity sha1-mjRBDk9OPaI96jdb5b5w8kd47Dk=
@@ -1928,11 +2326,24 @@
resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.3.2.tgz#a894b75d4bc4f6cd679ef3244a9fd8f46ae2d428"
integrity sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg=
+array.prototype.flat@^1.2.1:
+ version "1.2.3"
+ resolved "https://registry.yarnpkg.com/array.prototype.flat/-/array.prototype.flat-1.2.3.tgz#0de82b426b0318dbfdb940089e38b043d37f6c7b"
+ integrity sha512-gBlRZV0VSmfPIeWfuuy56XZMvbVfbEUnOXUvt3F/eUUUSyzlgLxhEX4YAEpxNAogRGehPSnfXyPtYyKAhkzQhQ==
+ dependencies:
+ define-properties "^1.1.3"
+ es-abstract "^1.17.0-next.1"
+
arrify@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/arrify/-/arrify-1.0.1.tgz#898508da2226f380df904728456849c1501a4b0d"
integrity sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=
+arrify@^2.0.1:
+ version "2.0.1"
+ resolved "https://registry.yarnpkg.com/arrify/-/arrify-2.0.1.tgz#c9655e9331e0abcd588d2a7cad7e9956f66701fa"
+ integrity sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==
+
asap@~2.0.3, asap@~2.0.6:
version "2.0.6"
resolved "https://registry.yarnpkg.com/asap/-/asap-2.0.6.tgz#e50347611d7e690943208bbdafebcbc2fb866d46"
@@ -1959,6 +2370,13 @@
resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525"
integrity sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=
+assert@1.4.1:
+ version "1.4.1"
+ resolved "https://registry.yarnpkg.com/assert/-/assert-1.4.1.tgz#99912d591836b5a6f5b345c0f07eefc08fc65d91"
+ integrity sha1-mZEtWRg2tab1s0XA8H7vwI/GXZE=
+ dependencies:
+ util "0.10.3"
+
assert@^1.1.1:
version "1.5.0"
resolved "https://registry.yarnpkg.com/assert/-/assert-1.5.0.tgz#55c109aaf6e0aefdb3dc4b71240c70bf574b18eb"
@@ -2009,23 +2427,28 @@
resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
integrity sha1-x57Zf380y48robyXkLzDZkdLS3k=
+at-least-node@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/at-least-node/-/at-least-node-1.0.0.tgz#602cd4b46e844ad4effc92a8011a3c46e0238dc2"
+ integrity sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==
+
atob@^2.1.2:
version "2.1.2"
resolved "https://registry.yarnpkg.com/atob/-/atob-2.1.2.tgz#6d9517eb9e030d2436666651e86bd9f6f13533c9"
integrity sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==
-autoprefixer@^9.4.9:
- version "9.7.3"
- resolved "https://registry.yarnpkg.com/autoprefixer/-/autoprefixer-9.7.3.tgz#fd42ed03f53de9beb4ca0d61fb4f7268a9bb50b4"
- integrity sha512-8T5Y1C5Iyj6PgkPSFd0ODvK9DIleuPKUPYniNxybS47g2k2wFgLZ46lGQHlBuGKIAEV8fbCDfKCCRS1tvOgc3Q==
+autoprefixer@^9.6.1:
+ version "9.7.6"
+ resolved "https://registry.yarnpkg.com/autoprefixer/-/autoprefixer-9.7.6.tgz#63ac5bbc0ce7934e6997207d5bb00d68fa8293a4"
+ integrity sha512-F7cYpbN7uVVhACZTeeIeealwdGM6wMtfWARVLTy5xmKtgVdBNJvbDRoCK3YO1orcs7gv/KwYlb3iXwu9Ug9BkQ==
dependencies:
- browserslist "^4.8.0"
- caniuse-lite "^1.0.30001012"
+ browserslist "^4.11.1"
+ caniuse-lite "^1.0.30001039"
chalk "^2.4.2"
normalize-range "^0.1.2"
num2fraction "^1.2.2"
- postcss "^7.0.23"
- postcss-value-parser "^4.0.2"
+ postcss "^7.0.27"
+ postcss-value-parser "^4.0.3"
aws-sign2@~0.7.0:
version "0.7.0"
@@ -2062,17 +2485,17 @@
esutils "^2.0.2"
js-tokens "^3.0.2"
-babel-eslint@10.0.1:
- version "10.0.1"
- resolved "https://registry.yarnpkg.com/babel-eslint/-/babel-eslint-10.0.1.tgz#919681dc099614cd7d31d45c8908695092a1faed"
- integrity sha512-z7OT1iNV+TjOwHNLLyJk+HN+YVWX+CLE6fPD2SymJZOZQBs+QIexFjhm4keGTm8MW9xr4EC9Q0PbaLB24V5GoQ==
+babel-eslint@10.1.0:
+ version "10.1.0"
+ resolved "https://registry.yarnpkg.com/babel-eslint/-/babel-eslint-10.1.0.tgz#6968e568a910b78fb3779cdd8b6ac2f479943232"
+ integrity sha512-ifWaTHQ0ce+448CYop8AdrQiBsGrnC+bMgfyKFdi6EsPLTAWG+QfyDeM6OH+FmWnKvEq5NnBMLvlBUPKQZoDSg==
dependencies:
"@babel/code-frame" "^7.0.0"
- "@babel/parser" "^7.0.0"
- "@babel/traverse" "^7.0.0"
- "@babel/types" "^7.0.0"
- eslint-scope "3.7.1"
+ "@babel/parser" "^7.7.0"
+ "@babel/traverse" "^7.7.0"
+ "@babel/types" "^7.7.0"
eslint-visitor-keys "^1.0.0"
+ resolve "^1.12.0"
babel-extract-comments@^1.0.0:
version "1.0.0"
@@ -2081,19 +2504,6 @@
dependencies:
babylon "^6.18.0"
-babel-jest@24.7.1:
- version "24.7.1"
- resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-24.7.1.tgz#73902c9ff15a7dfbdc9994b0b17fcefd96042178"
- integrity sha512-GPnLqfk8Mtt0i4OemjWkChi73A3ALs4w2/QbG64uAj8b5mmwzxc7jbJVRZt8NJkxi6FopVHog9S3xX6UJKb2qg==
- dependencies:
- "@jest/transform" "^24.7.1"
- "@jest/types" "^24.7.0"
- "@types/babel__core" "^7.1.0"
- babel-plugin-istanbul "^5.1.0"
- babel-preset-jest "^24.6.0"
- chalk "^2.4.2"
- slash "^2.0.0"
-
babel-jest@^24.9.0:
version "24.9.0"
resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-24.9.0.tgz#3fc327cb8467b89d14d7bc70e315104a783ccd54"
@@ -2107,27 +2517,21 @@
chalk "^2.4.2"
slash "^2.0.0"
-babel-loader@8.0.5:
- version "8.0.5"
- resolved "https://registry.yarnpkg.com/babel-loader/-/babel-loader-8.0.5.tgz#225322d7509c2157655840bba52e46b6c2f2fe33"
- integrity sha512-NTnHnVRd2JnRqPC0vW+iOQWU5pchDbYXsG2E6DMXEpMfUcQKclF9gmf3G3ZMhzG7IG9ji4coL0cm+FxeWxDpnw==
+babel-loader@8.1.0:
+ version "8.1.0"
+ resolved "https://registry.yarnpkg.com/babel-loader/-/babel-loader-8.1.0.tgz#c611d5112bd5209abe8b9fa84c3e4da25275f1c3"
+ integrity sha512-7q7nC1tYOrqvUrN3LQK4GwSk/TQorZSOlO9C+RZDZpODgyN4ZlCqE5q9cDsyWOliN+aU9B4JX01xK9eJXowJLw==
dependencies:
- find-cache-dir "^2.0.0"
- loader-utils "^1.0.2"
- mkdirp "^0.5.1"
- util.promisify "^1.0.0"
+ find-cache-dir "^2.1.0"
+ loader-utils "^1.4.0"
+ mkdirp "^0.5.3"
+ pify "^4.0.1"
+ schema-utils "^2.6.5"
-babel-plugin-dynamic-import-node@2.2.0:
- version "2.2.0"
- resolved "https://registry.yarnpkg.com/babel-plugin-dynamic-import-node/-/babel-plugin-dynamic-import-node-2.2.0.tgz#c0adfb07d95f4a4495e9aaac6ec386c4d7c2524e"
- integrity sha512-fP899ELUnTaBcIzmrW7nniyqqdYWrWuJUyPWHxFa/c7r7hS6KC8FscNfLlBNIoPSc55kYMGEEKjPjJGCLbE1qA==
- dependencies:
- object.assign "^4.1.0"
-
-babel-plugin-dynamic-import-node@^2.3.0:
- version "2.3.0"
- resolved "https://registry.yarnpkg.com/babel-plugin-dynamic-import-node/-/babel-plugin-dynamic-import-node-2.3.0.tgz#f00f507bdaa3c3e3ff6e7e5e98d90a7acab96f7f"
- integrity sha512-o6qFkpeQEBxcqt0XYlWzAVxNCSCZdUgcR8IRlhD/8DylxjjO4foPcvTW0GGKa/cVt3rvxZ7o5ippJ+/0nvLhlQ==
+babel-plugin-dynamic-import-node@^2.3.3:
+ version "2.3.3"
+ resolved "https://registry.yarnpkg.com/babel-plugin-dynamic-import-node/-/babel-plugin-dynamic-import-node-2.3.3.tgz#84fda19c976ec5c6defef57f9427b3def66e17a3"
+ integrity sha512-jZVI+s9Zg3IqA/kdi0i6UDCybUI3aSBLnglhYbSSjKlV7yF1F/5LWv8MakQmvYpnbJDS6fcBL2KzHSxNCMtWSQ==
dependencies:
object.assign "^4.1.0"
@@ -2156,19 +2560,19 @@
dependencies:
"@types/babel__traverse" "^7.0.6"
-babel-plugin-macros@2.5.1:
- version "2.5.1"
- resolved "https://registry.yarnpkg.com/babel-plugin-macros/-/babel-plugin-macros-2.5.1.tgz#4a119ac2c2e19b458c259b9accd7ee34fd57ec6f"
- integrity sha512-xN3KhAxPzsJ6OQTktCanNpIFnnMsCV+t8OloKxIL72D6+SUZYFn9qfklPgef5HyyDtzYZqqb+fs1S12+gQY82Q==
+babel-plugin-macros@2.8.0:
+ version "2.8.0"
+ resolved "https://registry.yarnpkg.com/babel-plugin-macros/-/babel-plugin-macros-2.8.0.tgz#0f958a7cc6556b1e65344465d99111a1e5e10138"
+ integrity sha512-SEP5kJpfGYqYKpBrj5XU3ahw5p5GOHJ0U5ssOSQ/WBVdwkD2Dzlce95exQTs3jOVWPPKLBN2rlEWkCK7dSmLvg==
dependencies:
- "@babel/runtime" "^7.4.2"
- cosmiconfig "^5.2.0"
- resolve "^1.10.0"
+ "@babel/runtime" "^7.7.2"
+ cosmiconfig "^6.0.0"
+ resolve "^1.12.0"
-babel-plugin-named-asset-import@^0.3.2:
- version "0.3.5"
- resolved "https://registry.yarnpkg.com/babel-plugin-named-asset-import/-/babel-plugin-named-asset-import-0.3.5.tgz#d3fa1a7f1f4babd4ed0785b75e2f926df0d70d0d"
- integrity sha512-sGhfINU+AuMw9oFAdIn/nD5sem3pn/WgxAfDZ//Q3CnF+5uaho7C7shh2rKLk6sKE/XkfmyibghocwKdVjLIKg==
+babel-plugin-named-asset-import@^0.3.6:
+ version "0.3.6"
+ resolved "https://registry.yarnpkg.com/babel-plugin-named-asset-import/-/babel-plugin-named-asset-import-0.3.6.tgz#c9750a1b38d85112c9e166bf3ef7c5dbc605f4be"
+ integrity sha512-1aGDUfL1qOOIoqk9QKGIo2lANk+C7ko/fqH0uIyC71x3PEGz0uVP8ISgfEsFuG+FKmjHTvFK/nNM8dowpmUxLA==
babel-plugin-syntax-object-rest-spread@^6.8.0:
version "6.13.0"
@@ -2188,7 +2592,7 @@
resolved "https://registry.yarnpkg.com/babel-plugin-transform-react-remove-prop-types/-/babel-plugin-transform-react-remove-prop-types-0.4.24.tgz#f2edaf9b4c6a5fbe5c1d678bfb531078c1555f3a"
integrity sha512-eqj0hVcJUR57/Ug2zE1Yswsw4LhuqqHhD+8v120T1cl3kjg76QwtyBrdIk4WVwK+lAhBJVYCd/v+4nc4y+8JsA==
-babel-preset-jest@^24.6.0, babel-preset-jest@^24.9.0:
+babel-preset-jest@^24.9.0:
version "24.9.0"
resolved "https://registry.yarnpkg.com/babel-preset-jest/-/babel-preset-jest-24.9.0.tgz#192b521e2217fb1d1f67cf73f70c336650ad3cdc"
integrity sha512-izTUuhE4TMfTRPF92fFwD2QfdXaZW08qvWTFCI51V8rW5x00UuPgc3ajRoWofXOuxjfcOM5zzSYsQS3H8KGCAg==
@@ -2196,28 +2600,25 @@
"@babel/plugin-syntax-object-rest-spread" "^7.0.0"
babel-plugin-jest-hoist "^24.9.0"
-babel-preset-react-app@^8.0.0:
- version "8.0.0"
- resolved "https://registry.yarnpkg.com/babel-preset-react-app/-/babel-preset-react-app-8.0.0.tgz#930b6e28cdcfdff97ddb8bef9226d504f244d326"
- integrity sha512-6Dmj7e8l7eWE+R6sKKLRrGEQXMfcBqBYlphaAgT1ml8qT1NEP+CyTZyfjmgKGqHZfwH3RQCUOuP6y4mpGc7tgg==
+babel-preset-react-app@^9.1.2:
+ version "9.1.2"
+ resolved "https://registry.yarnpkg.com/babel-preset-react-app/-/babel-preset-react-app-9.1.2.tgz#54775d976588a8a6d1a99201a702befecaf48030"
+ integrity sha512-k58RtQOKH21NyKtzptoAvtAODuAJJs3ZhqBMl456/GnXEQ/0La92pNmwgWoMn5pBTrsvk3YYXdY7zpY4e3UIxA==
dependencies:
- "@babel/core" "7.4.3"
- "@babel/plugin-proposal-class-properties" "7.4.0"
- "@babel/plugin-proposal-decorators" "7.4.0"
- "@babel/plugin-proposal-object-rest-spread" "7.4.3"
- "@babel/plugin-syntax-dynamic-import" "7.2.0"
- "@babel/plugin-transform-classes" "7.4.3"
- "@babel/plugin-transform-destructuring" "7.4.3"
- "@babel/plugin-transform-flow-strip-types" "7.4.0"
- "@babel/plugin-transform-react-constant-elements" "7.2.0"
- "@babel/plugin-transform-react-display-name" "7.2.0"
- "@babel/plugin-transform-runtime" "7.4.3"
- "@babel/preset-env" "7.4.3"
- "@babel/preset-react" "7.0.0"
- "@babel/preset-typescript" "7.3.3"
- "@babel/runtime" "7.4.3"
- babel-plugin-dynamic-import-node "2.2.0"
- babel-plugin-macros "2.5.1"
+ "@babel/core" "7.9.0"
+ "@babel/plugin-proposal-class-properties" "7.8.3"
+ "@babel/plugin-proposal-decorators" "7.8.3"
+ "@babel/plugin-proposal-nullish-coalescing-operator" "7.8.3"
+ "@babel/plugin-proposal-numeric-separator" "7.8.3"
+ "@babel/plugin-proposal-optional-chaining" "7.9.0"
+ "@babel/plugin-transform-flow-strip-types" "7.9.0"
+ "@babel/plugin-transform-react-display-name" "7.8.3"
+ "@babel/plugin-transform-runtime" "7.9.0"
+ "@babel/preset-env" "7.9.0"
+ "@babel/preset-react" "7.9.1"
+ "@babel/preset-typescript" "7.9.0"
+ "@babel/runtime" "7.9.0"
+ babel-plugin-macros "2.8.0"
babel-plugin-transform-react-remove-prop-types "0.4.24"
babel-runtime@6.x, babel-runtime@^6.23.0, babel-runtime@^6.26.0:
@@ -2285,6 +2686,11 @@
resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-1.13.1.tgz#598afe54755b2868a5330d2aff9d4ebb53209b65"
integrity sha512-Un7MIEDdUC5gNpcGDV97op1Ywk748MpHcFTHoYs6qnj1Z3j7I53VG3nwZhKzoBZmbdRNnb6WRdFlwl7tSDuZGw==
+binary-extensions@^2.0.0:
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.0.0.tgz#23c0df14f6a88077f5f986c0d167ec03c3d5537c"
+ integrity sha512-Phlt0plgpIIBOGTT/ehfFnbNlfsDEiqmzE2KRXoX1bLIlir4X/MR+zSyBEkL05ffWgnRSf/DXv+WrUAVr93/ow==
+
bindings@^1.5.0:
version "1.5.0"
resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df"
@@ -2349,6 +2755,20 @@
type-fest "^0.3.0"
widest-line "^2.0.0"
+boxen@^4.2.0:
+ version "4.2.0"
+ resolved "https://registry.yarnpkg.com/boxen/-/boxen-4.2.0.tgz#e411b62357d6d6d36587c8ac3d5d974daa070e64"
+ integrity sha512-eB4uT9RGzg2odpER62bBwSLvUeGC+WbRjjyyFhGsKnc8wp/m0+hQsMUvUe3H2V0D5vw0nBdO1hCJoZo5mKeuIQ==
+ dependencies:
+ ansi-align "^3.0.0"
+ camelcase "^5.3.1"
+ chalk "^3.0.0"
+ cli-boxes "^2.2.0"
+ string-width "^4.1.0"
+ term-size "^2.1.0"
+ type-fest "^0.8.1"
+ widest-line "^3.1.0"
+
brace-expansion@^1.1.7:
version "1.1.11"
resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd"
@@ -2373,6 +2793,13 @@
split-string "^3.0.2"
to-regex "^3.0.1"
+braces@^3.0.1, braces@~3.0.2:
+ version "3.0.2"
+ resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107"
+ integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==
+ dependencies:
+ fill-range "^7.0.1"
+
brorand@^1.0.1:
version "1.1.0"
resolved "https://registry.yarnpkg.com/brorand/-/brorand-1.1.0.tgz#12c25efe40a45e3c323eb8675a0a0ce57b22371f"
@@ -2449,16 +2876,17 @@
dependencies:
pako "~1.0.5"
-browserslist@4.7.0:
- version "4.7.0"
- resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.7.0.tgz#9ee89225ffc07db03409f2fee524dc8227458a17"
- integrity sha512-9rGNDtnj+HaahxiVV38Gn8n8Lr8REKsel68v1sPFfIGEK6uSXTY3h9acgiT1dZVtOOUtifo/Dn8daDQ5dUgVsA==
+browserslist@4.10.0:
+ version "4.10.0"
+ resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.10.0.tgz#f179737913eaf0d2b98e4926ac1ca6a15cbcc6a9"
+ integrity sha512-TpfK0TDgv71dzuTsEAlQiHeWQ/tiPqgNZVdv046fvNtBZrjbv2O3TsWCDU0AWGJJKCF/KsjNdLzR9hXOsh/CfA==
dependencies:
- caniuse-lite "^1.0.30000989"
- electron-to-chromium "^1.3.247"
- node-releases "^1.1.29"
+ caniuse-lite "^1.0.30001035"
+ electron-to-chromium "^1.3.378"
+ node-releases "^1.1.52"
+ pkg-up "^3.1.0"
-browserslist@^4.0.0, browserslist@^4.1.1, browserslist@^4.4.2, browserslist@^4.5.2, browserslist@^4.6.0, browserslist@^4.8.0, browserslist@^4.8.3:
+browserslist@^4.0.0:
version "4.8.3"
resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.8.3.tgz#65802fcd77177c878e015f0e3189f2c4f627ba44"
integrity sha512-iU43cMMknxG1ClEZ2MDKeonKE1CCrFVkQK2AqO2YWFmvIrx4JWrvQ4w4hQez6EpVI8rHTtqh/ruHHDHSOKxvUg==
@@ -2467,6 +2895,16 @@
electron-to-chromium "^1.3.322"
node-releases "^1.1.44"
+browserslist@^4.11.1, browserslist@^4.6.2, browserslist@^4.6.4, browserslist@^4.8.5, browserslist@^4.9.1:
+ version "4.12.0"
+ resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.12.0.tgz#06c6d5715a1ede6c51fc39ff67fd647f740b656d"
+ integrity sha512-UH2GkcEDSI0k/lRkuDSzFl9ZZ87skSy9w2XAn1MsZnL+4c4rqbBd3e82UWHbYDpztABrPBhZsTEeuxVfHppqDg==
+ dependencies:
+ caniuse-lite "^1.0.30001043"
+ electron-to-chromium "^1.3.413"
+ node-releases "^1.1.53"
+ pkg-up "^2.0.0"
+
bser@2.1.1:
version "2.1.1"
resolved "https://registry.yarnpkg.com/bser/-/bser-2.1.1.tgz#e6787da20ece9d07998533cfd9de6f5c38f4bc05"
@@ -2474,6 +2912,11 @@
dependencies:
node-int64 "^0.4.0"
+buf-compare@^1.0.0:
+ version "1.0.1"
+ resolved "https://registry.yarnpkg.com/buf-compare/-/buf-compare-1.0.1.tgz#fef28da8b8113a0a0db4430b0b6467b69730b34a"
+ integrity sha1-/vKNqLgROgoNtEMLC2Rntpcws0o=
+
buffer-from@^1.0.0:
version "1.1.1"
resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef"
@@ -2513,26 +2956,6 @@
resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.0.tgz#f6cf7933a360e0588fa9fde85651cdc7f805d1f6"
integrity sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==
-cacache@^11.0.2:
- version "11.3.3"
- resolved "https://registry.yarnpkg.com/cacache/-/cacache-11.3.3.tgz#8bd29df8c6a718a6ebd2d010da4d7972ae3bbadc"
- integrity sha512-p8WcneCytvzPxhDvYp31PD039vi77I12W+/KfR9S8AZbaiARFBCpsPJS+9uhWfeBfeAtW7o/4vt3MUqLkbY6nA==
- dependencies:
- bluebird "^3.5.5"
- chownr "^1.1.1"
- figgy-pudding "^3.5.1"
- glob "^7.1.4"
- graceful-fs "^4.1.15"
- lru-cache "^5.1.1"
- mississippi "^3.0.0"
- mkdirp "^0.5.1"
- move-concurrently "^1.0.1"
- promise-inflight "^1.0.1"
- rimraf "^2.6.3"
- ssri "^6.0.1"
- unique-filename "^1.1.1"
- y18n "^4.0.0"
-
cacache@^12.0.2:
version "12.0.3"
resolved "https://registry.yarnpkg.com/cacache/-/cacache-12.0.3.tgz#be99abba4e1bf5df461cd5a2c1071fc432573390"
@@ -2554,6 +2977,30 @@
unique-filename "^1.1.1"
y18n "^4.0.0"
+cacache@^13.0.1:
+ version "13.0.1"
+ resolved "https://registry.yarnpkg.com/cacache/-/cacache-13.0.1.tgz#a8000c21697089082f85287a1aec6e382024a71c"
+ integrity sha512-5ZvAxd05HDDU+y9BVvcqYu2LLXmPnQ0hW62h32g4xBTgL/MppR4/04NHfj/ycM2y6lmTnbw6HVi+1eN0Psba6w==
+ dependencies:
+ chownr "^1.1.2"
+ figgy-pudding "^3.5.1"
+ fs-minipass "^2.0.0"
+ glob "^7.1.4"
+ graceful-fs "^4.2.2"
+ infer-owner "^1.0.4"
+ lru-cache "^5.1.1"
+ minipass "^3.0.0"
+ minipass-collect "^1.0.2"
+ minipass-flush "^1.0.5"
+ minipass-pipeline "^1.2.2"
+ mkdirp "^0.5.1"
+ move-concurrently "^1.0.1"
+ p-map "^3.0.0"
+ promise-inflight "^1.0.1"
+ rimraf "^2.7.1"
+ ssri "^7.0.0"
+ unique-filename "^1.1.1"
+
cache-base@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/cache-base/-/cache-base-1.0.1.tgz#0a7f46416831c8b662ee36fe4e7c59d76f666ab2"
@@ -2611,24 +3058,38 @@
resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73"
integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==
-camel-case@3.0.x:
- version "3.0.0"
- resolved "https://registry.yarnpkg.com/camel-case/-/camel-case-3.0.0.tgz#ca3c3688a4e9cf3a4cda777dc4dcbc713249cf73"
- integrity sha1-yjw2iKTpzzpM2nd9xNy8cTJJz3M=
+camel-case@^4.1.1:
+ version "4.1.1"
+ resolved "https://registry.yarnpkg.com/camel-case/-/camel-case-4.1.1.tgz#1fc41c854f00e2f7d0139dfeba1542d6896fe547"
+ integrity sha512-7fa2WcG4fYFkclIvEmxBbTvmibwF2/agfEBc6q3lOpVu0A13ltLsA+Hr/8Hp6kp5f+G7hKi6t8lys6XxP+1K6Q==
dependencies:
- no-case "^2.2.0"
- upper-case "^1.1.1"
+ pascal-case "^3.1.1"
+ tslib "^1.10.0"
+
+camelcase-keys@^4.0.0:
+ version "4.2.0"
+ resolved "https://registry.yarnpkg.com/camelcase-keys/-/camelcase-keys-4.2.0.tgz#a2aa5fb1af688758259c32c141426d78923b9b77"
+ integrity sha1-oqpfsa9oh1glnDLBQUJteJI7m3c=
+ dependencies:
+ camelcase "^4.1.0"
+ map-obj "^2.0.0"
+ quick-lru "^1.0.0"
+
+camelcase@5.0.0:
+ version "5.0.0"
+ resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.0.0.tgz#03295527d58bd3cd4aa75363f35b2e8d97be2f42"
+ integrity sha512-faqwZqnWxbxn+F1d399ygeamQNy3lPp/H9H6rNrqYh4FSVCtcY+3cub1MxA8o9mDd55mM8Aghuu/kuyYA6VTsA==
+
+camelcase@5.3.1, camelcase@^5.0.0, camelcase@^5.3.1:
+ version "5.3.1"
+ resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.3.1.tgz#e3c9b31569e106811df242f715725a1f4c494320"
+ integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==
camelcase@^4.1.0:
version "4.1.0"
resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-4.1.0.tgz#d545635be1e33c542649c69173e5de6acfae34dd"
integrity sha1-1UVjW+HjPFQmScaRc+Xeas+uNN0=
-camelcase@^5.0.0, camelcase@^5.2.0, camelcase@^5.3.1:
- version "5.3.1"
- resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.3.1.tgz#e3c9b31569e106811df242f715725a1f4c494320"
- integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==
-
caniuse-api@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/caniuse-api/-/caniuse-api-3.0.0.tgz#5e4d90e2274961d46291997df599e3ed008ee4c0"
@@ -2639,11 +3100,16 @@
lodash.memoize "^4.1.2"
lodash.uniq "^4.5.0"
-caniuse-lite@^1.0.0, caniuse-lite@^1.0.30000939, caniuse-lite@^1.0.30000989, caniuse-lite@^1.0.30001012, caniuse-lite@^1.0.30001017:
+caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001017:
version "1.0.30001019"
resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001019.tgz#857e3fccaad2b2feb3f1f6d8a8f62d747ea648e1"
integrity sha512-6ljkLtF1KM5fQ+5ZN0wuyVvvebJxgJPTmScOMaFuQN2QuOzvRJnWSKfzQskQU5IOU4Gap3zasYPIinzwUjoj/g==
+caniuse-lite@^1.0.30000981, caniuse-lite@^1.0.30001035, caniuse-lite@^1.0.30001039, caniuse-lite@^1.0.30001043:
+ version "1.0.30001054"
+ resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001054.tgz#7e82fc42d927980b0ce1426c4813df12381e1a75"
+ integrity sha512-jiKlTI6Ur8Kjfj8z0muGrV6FscpRvefcQVPSuMuXnvRCfExU7zlVLNjmOz1TnurWgUrAY7MMmjyy+uTgIl1XHw==
+
capture-exit@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/capture-exit/-/capture-exit-2.0.0.tgz#fb953bfaebeb781f62898239dabb426d08a509a4"
@@ -2651,10 +3117,10 @@
dependencies:
rsvp "^4.8.4"
-case-sensitive-paths-webpack-plugin@2.2.0:
- version "2.2.0"
- resolved "https://registry.yarnpkg.com/case-sensitive-paths-webpack-plugin/-/case-sensitive-paths-webpack-plugin-2.2.0.tgz#3371ef6365ef9c25fa4b81c16ace0e9c7dc58c3e"
- integrity sha512-u5ElzokS8A1pm9vM3/iDgTcI3xqHxuCao94Oz8etI3cf0Tio0p8izkDYbTIn09uP3yUUr6+veaE6IkjnTYS46g==
+case-sensitive-paths-webpack-plugin@2.3.0:
+ version "2.3.0"
+ resolved "https://registry.yarnpkg.com/case-sensitive-paths-webpack-plugin/-/case-sensitive-paths-webpack-plugin-2.3.0.tgz#23ac613cc9a856e4f88ff8bb73bbb5e989825cf7"
+ integrity sha512-/4YgnZS8y1UXXmC02xD5rRrBEu6T5ub+mQHLNRj0fzTRbgdBYhsNo2V5EqwgqrExjxsjtF/OpAKAMkKsxbD5XQ==
caseless@~0.12.0:
version "0.12.0"
@@ -2681,12 +3147,20 @@
strip-ansi "^3.0.0"
supports-color "^2.0.0"
+chalk@^3.0.0:
+ version "3.0.0"
+ resolved "https://registry.yarnpkg.com/chalk/-/chalk-3.0.0.tgz#3f73c2bf526591f574cc492c51e2456349f844e4"
+ integrity sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==
+ dependencies:
+ ansi-styles "^4.1.0"
+ supports-color "^7.1.0"
+
chardet@^0.7.0:
version "0.7.0"
resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e"
integrity sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==
-chokidar@^2.0.0, chokidar@^2.0.2, chokidar@^2.0.4:
+chokidar@^2.1.8:
version "2.1.8"
resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-2.1.8.tgz#804b3a7b6a99358c3c5c61e71d8728f041cff917"
integrity sha512-ZmZUazfOzf0Nve7duiCKD23PFSCs4JPoYyccjUFF3aQkQadqBhfzhjkwBH2mNOG9cTBwhamM37EIsIkZw3nRgg==
@@ -2705,12 +3179,32 @@
optionalDependencies:
fsevents "^1.2.7"
+chokidar@^3.3.0:
+ version "3.4.0"
+ resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.4.0.tgz#b30611423ce376357c765b9b8f904b9fba3c0be8"
+ integrity sha512-aXAaho2VJtisB/1fg1+3nlLJqGOuewTzQpd/Tz0yTg2R0e4IGtshYvtjowyEumcBv2z+y4+kc75Mz7j5xJskcQ==
+ dependencies:
+ anymatch "~3.1.1"
+ braces "~3.0.2"
+ glob-parent "~5.1.0"
+ is-binary-path "~2.1.0"
+ is-glob "~4.0.1"
+ normalize-path "~3.0.0"
+ readdirp "~3.4.0"
+ optionalDependencies:
+ fsevents "~2.1.2"
+
chownr@^1.1.1:
version "1.1.3"
resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.3.tgz#42d837d5239688d55f303003a508230fa6727142"
integrity sha512-i70fVHhmV3DtTl6nqvZOnIjbY0Pe4kAUjwHj8z0zAdgBtYrJyYwLKCCuRBQ5ppkyL0AkN7HKRnETdmdp1zqNXw==
-chrome-trace-event@^1.0.0:
+chownr@^1.1.2:
+ version "1.1.4"
+ resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b"
+ integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==
+
+chrome-trace-event@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/chrome-trace-event/-/chrome-trace-event-1.0.2.tgz#234090ee97c7d4ad1a2c4beae27505deffc608a4"
integrity sha512-9e/zx1jw7B4CO+c/RXoCsfg/x1AfUBioy4owYH0bJprEYAx5hRFLRhWBqHAG57D0ZM4H7vxbP7bPe0VwhQRYDQ==
@@ -2745,24 +3239,36 @@
resolved "https://registry.yarnpkg.com/classnames/-/classnames-2.2.6.tgz#43935bffdd291f326dad0a205309b38d00f650ce"
integrity sha512-JR/iSQOSt+LQIWwrwEzJ9uk0xfN3mTVYMwt1Ir5mUcSN6pU+V4zQFFaJsclJbPuAUQH+yfWef6tm7l1quW3C8Q==
-clean-css@4.2.x:
- version "4.2.1"
- resolved "https://registry.yarnpkg.com/clean-css/-/clean-css-4.2.1.tgz#2d411ef76b8569b6d0c84068dabe85b0aa5e5c17"
- integrity sha512-4ZxI6dy4lrY6FHzfiy1aEOXgu4LIsW2MhwG0VBKdcoGoH/XLFgaHSdLTGr4O8Be6A8r3MOphEiI8Gc1n0ecf3g==
+clean-css@^4.2.3:
+ version "4.2.3"
+ resolved "https://registry.yarnpkg.com/clean-css/-/clean-css-4.2.3.tgz#507b5de7d97b48ee53d84adb0160ff6216380f78"
+ integrity sha512-VcMWDN54ZN/DS+g58HYL5/n4Zrqe8vHJpGA8KdgUXFU4fuP/aHNw8eld9SyEIyabIMJX/0RaY/fplOo5hYLSFA==
dependencies:
source-map "~0.6.0"
+clean-regexp@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/clean-regexp/-/clean-regexp-1.0.0.tgz#8df7c7aae51fd36874e8f8d05b9180bc11a3fed7"
+ integrity sha1-jffHquUf02h06PjQW5GAvBGj/tc=
+ dependencies:
+ escape-string-regexp "^1.0.5"
+
+clean-stack@^2.0.0:
+ version "2.2.0"
+ resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b"
+ integrity sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==
+
cli-boxes@^2.2.0:
version "2.2.0"
resolved "https://registry.yarnpkg.com/cli-boxes/-/cli-boxes-2.2.0.tgz#538ecae8f9c6ca508e3c3c95b453fe93cb4c168d"
integrity sha512-gpaBrMAizVEANOpfZp/EEUixTXDyGt7DFzdK5hU+UbWt/J0lB0w20ncZj59Z9a93xHb9u12zF5BS6i9RKbtg4w==
-cli-cursor@^2.1.0:
- version "2.1.0"
- resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-2.1.0.tgz#b35dac376479facc3e94747d41d0d0f5238ffcb5"
- integrity sha1-s12sN2R5+sw+lHR9QdDQ9SOP/LU=
+cli-cursor@^3.1.0:
+ version "3.1.0"
+ resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-3.1.0.tgz#264305a7ae490d1d03bf0c9ba7c925d1753af307"
+ integrity sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==
dependencies:
- restore-cursor "^2.0.0"
+ restore-cursor "^3.1.0"
cli-width@^2.0.0:
version "2.2.0"
@@ -2798,15 +3304,14 @@
lazy-cache "^1.0.3"
shallow-clone "^0.1.2"
-clone-deep@^2.0.1:
- version "2.0.2"
- resolved "https://registry.yarnpkg.com/clone-deep/-/clone-deep-2.0.2.tgz#00db3a1e173656730d1188c3d6aced6d7ea97713"
- integrity sha512-SZegPTKjCgpQH63E+eN6mVEEPdQBOUzjyJm5Pora4lrwWRFS8I0QAxV/KD6vV/i0WuijHZWQC1fMsPEdxfdVCQ==
+clone-deep@^4.0.1:
+ version "4.0.1"
+ resolved "https://registry.yarnpkg.com/clone-deep/-/clone-deep-4.0.1.tgz#c19fd9bdbbf85942b4fd979c84dcf7d5f07c2387"
+ integrity sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==
dependencies:
- for-own "^1.0.0"
is-plain-object "^2.0.4"
- kind-of "^6.0.0"
- shallow-clone "^1.0.0"
+ kind-of "^6.0.2"
+ shallow-clone "^3.0.0"
clone-response@^1.0.2:
version "1.0.2"
@@ -2854,12 +3359,19 @@
dependencies:
color-name "1.1.3"
+color-convert@^2.0.1:
+ version "2.0.1"
+ resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-2.0.1.tgz#72d3a68d598c9bdb3af2ad1e84f21d896abd4de3"
+ integrity sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==
+ dependencies:
+ color-name "~1.1.4"
+
color-name@1.1.3:
version "1.1.3"
resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25"
integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=
-color-name@^1.0.0:
+color-name@^1.0.0, color-name@~1.1.4:
version "1.1.4"
resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2"
integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==
@@ -2887,20 +3399,15 @@
dependencies:
delayed-stream "~1.0.0"
-commander@2.17.x:
- version "2.17.1"
- resolved "https://registry.yarnpkg.com/commander/-/commander-2.17.1.tgz#bd77ab7de6de94205ceacc72f1716d29f20a77bf"
- integrity sha512-wPMUt6FnH2yzG95SA6mzjQOEKUU3aLaDEmzs1ti+1E9h+CsrZghRlqEM/EJ4KscsQVG8uNN4uVreUeT8+drlgg==
-
-commander@^2.11.0, commander@^2.19.0, commander@^2.20.0, commander@~2.20.3:
+commander@^2.11.0, commander@^2.20.0, commander@~2.20.3:
version "2.20.3"
resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33"
integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==
-commander@~2.19.0:
- version "2.19.0"
- resolved "https://registry.yarnpkg.com/commander/-/commander-2.19.0.tgz#f6198aa84e5b83c46054b94ddedbfed5ee9ff12a"
- integrity sha512-6tvAOO+D6OENvRAh524Dh9jcfKTYDQAqvqezbCW82xj5X0pSrcpxtvRKHLG0yBY6SD7PSDrJaj+0AiOcKVd1Xg==
+commander@^4.1.1:
+ version "4.1.1"
+ resolved "https://registry.yarnpkg.com/commander/-/commander-4.1.1.tgz#9fd602bd936294e9e9ef46a3f4d6964044b18068"
+ integrity sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==
common-tags@^1.8.0:
version "1.8.0"
@@ -2929,6 +3436,13 @@
resolved "https://registry.yarnpkg.com/component-indexof/-/component-indexof-0.0.3.tgz#11d091312239eb8f32c8f25ae9cb002ffe8d3c24"
integrity sha1-EdCRMSI5648yyPJa6csAL/6NPCQ=
+compose-function@3.0.3:
+ version "3.0.3"
+ resolved "https://registry.yarnpkg.com/compose-function/-/compose-function-3.0.3.tgz#9ed675f13cc54501d30950a486ff6a7ba3ab185f"
+ integrity sha1-ntZ18TzFRQHTCVCkhv9qe6OrGF8=
+ dependencies:
+ arity-n "^1.0.4"
+
compressible@~2.0.16:
version "2.0.18"
resolved "https://registry.yarnpkg.com/compressible/-/compressible-2.0.18.tgz#af53cca6b070d4c3c0750fbd77286a6d7cc46fba"
@@ -2936,7 +3450,7 @@
dependencies:
mime-db ">= 1.43.0 < 2"
-compression@^1.5.2, compression@^1.7.4:
+compression@^1.7.4:
version "1.7.4"
resolved "https://registry.yarnpkg.com/compression/-/compression-1.7.4.tgz#95523eff170ca57c29a0ca41e6fe131f41e5bb8f"
integrity sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ==
@@ -2976,12 +3490,24 @@
write-file-atomic "^2.0.0"
xdg-basedir "^3.0.0"
-confusing-browser-globals@^1.0.7:
+configstore@^5.0.1:
+ version "5.0.1"
+ resolved "https://registry.yarnpkg.com/configstore/-/configstore-5.0.1.tgz#d365021b5df4b98cdd187d6a3b0e3f6a7cc5ed96"
+ integrity sha512-aMKprgk5YhBNyH25hj8wGt2+D52Sw1DRRIzqBwLp2Ya9mFmY8KPvvtvmna8SxVR9JMZ4kzMD68N22vlaRpkeFA==
+ dependencies:
+ dot-prop "^5.2.0"
+ graceful-fs "^4.1.2"
+ make-dir "^3.0.0"
+ unique-string "^2.0.0"
+ write-file-atomic "^3.0.0"
+ xdg-basedir "^4.0.0"
+
+confusing-browser-globals@1.0.9, confusing-browser-globals@^1.0.9:
version "1.0.9"
resolved "https://registry.yarnpkg.com/confusing-browser-globals/-/confusing-browser-globals-1.0.9.tgz#72bc13b483c0276801681871d4898516f8f54fdd"
integrity sha512-KbS1Y0jMtyPgIxjO7ZzMAuUpAKMt1SzCL9fsrKsX6b0zJPTaT0SiSPmewwVZg9UAO83HVIlEhZF84LIjZ0lmAw==
-connect-history-api-fallback@^1.3.0:
+connect-history-api-fallback@^1.6.0:
version "1.6.0"
resolved "https://registry.yarnpkg.com/connect-history-api-fallback/-/connect-history-api-fallback-1.6.0.tgz#8b32089359308d111115d81cad3fceab888f97bc"
integrity sha512-e54B99q/OUoH64zYYRf3HBP5z24G38h5D3qXu23JGRoigpX5Ss4r9ZnDk3g0Z8uQC2x2lPaJ+UlWBc1ZWBWdLg==
@@ -3018,13 +3544,18 @@
resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b"
integrity sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==
-convert-source-map@^1.1.0, convert-source-map@^1.4.0, convert-source-map@^1.7.0:
+convert-source-map@1.7.0, convert-source-map@^1.4.0, convert-source-map@^1.7.0:
version "1.7.0"
resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.7.0.tgz#17a2cb882d7f77d3490585e2ce6c524424a3a442"
integrity sha512-4FJkXzKXEDB1snCFZlLP4gpC3JILicCpGbzG9f9G7tGqGCzETQ2hWPrcinA9oU4wtf2biUaEH5065UnMeR33oA==
dependencies:
safe-buffer "~5.1.1"
+convert-source-map@^0.3.3:
+ version "0.3.5"
+ resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-0.3.5.tgz#f1d802950af7dd2631a1febe0596550c86ab3190"
+ integrity sha1-8dgClQr33SYxof6+BZZVDIarMZA=
+
cookie-signature@1.0.6:
version "1.0.6"
resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c"
@@ -3059,12 +3590,20 @@
dependencies:
toggle-selection "^1.0.6"
-core-js-compat@^3.0.0, core-js-compat@^3.6.0:
- version "3.6.2"
- resolved "https://registry.yarnpkg.com/core-js-compat/-/core-js-compat-3.6.2.tgz#314ca8b84d5e71c27c19f1ecda966501b1cf1f10"
- integrity sha512-+G28dzfYGtAM+XGvB1C5AS1ZPKfQ47HLhcdeIQdZgQnJVdp7/D0m+W/TErwhgsX6CujRUk/LebB6dCrKrtJrvQ==
+core-assert@^0.2.0:
+ version "0.2.1"
+ resolved "https://registry.yarnpkg.com/core-assert/-/core-assert-0.2.1.tgz#f85e2cf9bfed28f773cc8b3fa5c5b69bdc02fe3f"
+ integrity sha1-+F4s+b/tKPdzzIs/pcW2m9wC/j8=
dependencies:
- browserslist "^4.8.3"
+ buf-compare "^1.0.0"
+ is-error "^2.2.0"
+
+core-js-compat@^3.6.2:
+ version "3.6.5"
+ resolved "https://registry.yarnpkg.com/core-js-compat/-/core-js-compat-3.6.5.tgz#2a51d9a4e25dfd6e690251aa81f99e3c05481f1c"
+ integrity sha512-7ItTKOhOZbznhXAQ2g/slGg1PJV5zDO/WdkTwi7UEOJmkvsE32PWvx6mKtDjiMpjnR2CNf6BAD6sSxIlv7ptng==
+ dependencies:
+ browserslist "^4.8.5"
semver "7.0.0"
core-js-pure@^3.0.0:
@@ -3082,10 +3621,10 @@
resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.6.11.tgz#38831469f9922bded8ee21c9dc46985e0399308c"
integrity sha512-5wjnpaT/3dV+XB4borEsnAYQchn00XSgTAWKDkEqv+K8KevjbzmofK6hfJ9TZIlpj2N0xQpazy7PiRQiWHqzWg==
-core-js@^3.4.1:
- version "3.6.2"
- resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.6.2.tgz#2799ea1a59050f0acf50dfe89b916d6503b16caa"
- integrity sha512-hIE5dXkRzRvnZ5vhkRfQxUvDxQZmD9oueA08jDYRBKJHx+VIl/Pne/e0A4x9LObEEthC/TqiZybUoNM4tRgnKg==
+core-js@^3.5.0:
+ version "3.6.5"
+ resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.6.5.tgz#7395dc273af37fb2e50e9bd3d9fe841285231d1a"
+ integrity sha512-vZVEEwZoIsI+vPEuoF9Iqf5H7/M3eeQqWlQnYa8FSKKePuYTf5MWnxb5SDAzCa60b3JBRS5g9b+Dq7b1y/RCrA==
core-util-is@1.0.2, core-util-is@~1.0.0:
version "1.0.2"
@@ -3100,7 +3639,7 @@
object-assign "^4"
vary "^1"
-cosmiconfig@^5.0.0, cosmiconfig@^5.2.0, cosmiconfig@^5.2.1:
+cosmiconfig@^5.0.0, cosmiconfig@^5.2.1:
version "5.2.1"
resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-5.2.1.tgz#040f726809c591e77a17c0a3626ca45b4f168b1a"
integrity sha512-H65gsXo1SKjf8zmrJ67eJk8aIRKV5ff2D4uKZIBZShbhGSpEmsQOPW/SKMKYhSTrqR7ufy6RP69rPogdaPh/kA==
@@ -3110,6 +3649,17 @@
js-yaml "^3.13.1"
parse-json "^4.0.0"
+cosmiconfig@^6.0.0:
+ version "6.0.0"
+ resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-6.0.0.tgz#da4fee853c52f6b1e6935f41c1a2fc50bd4a9982"
+ integrity sha512-xb3ZL6+L8b9JLLCx3ZdoZy4+2ECphCMo2PwqgP1tlfVq6M6YReyzBJtvWWtbDSpNr9hn96pkCiZqUcFEc+54Qg==
+ dependencies:
+ "@types/parse-json" "^4.0.0"
+ import-fresh "^3.1.0"
+ parse-json "^5.0.0"
+ path-type "^4.0.0"
+ yaml "^1.7.2"
+
create-ecdh@^4.0.0:
version "4.0.3"
resolved "https://registry.yarnpkg.com/create-ecdh/-/create-ecdh-4.0.3.tgz#c9111b6f33045c4697f144787f9254cdc77c45ff"
@@ -3150,16 +3700,14 @@
loose-envify "^1.3.1"
object-assign "^4.1.1"
-cross-spawn@6.0.5, cross-spawn@^6.0.0, cross-spawn@^6.0.5:
- version "6.0.5"
- resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4"
- integrity sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==
+cross-spawn@7.0.1:
+ version "7.0.1"
+ resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.1.tgz#0ab56286e0f7c24e153d04cc2aa027e43a9a5d14"
+ integrity sha512-u7v4o84SwFpD32Z8IIcPZ6z1/ie24O6RU3RbtL5Y316l3KuHVPx9ItBgWQ6VlfAFnRnTtMUrsQ9MUUTuEZjogg==
dependencies:
- nice-try "^1.0.4"
- path-key "^2.0.1"
- semver "^5.5.0"
- shebang-command "^1.2.0"
- which "^1.2.9"
+ path-key "^3.1.0"
+ shebang-command "^2.0.0"
+ which "^2.0.1"
cross-spawn@^5.0.1:
version "5.1.0"
@@ -3170,6 +3718,17 @@
shebang-command "^1.2.0"
which "^1.2.9"
+cross-spawn@^6.0.0, cross-spawn@^6.0.5:
+ version "6.0.5"
+ resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4"
+ integrity sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==
+ dependencies:
+ nice-try "^1.0.4"
+ path-key "^2.0.1"
+ semver "^5.5.0"
+ shebang-command "^1.2.0"
+ which "^1.2.9"
+
crypto-browserify@^3.11.0:
version "3.12.0"
resolved "https://registry.yarnpkg.com/crypto-browserify/-/crypto-browserify-3.12.0.tgz#396cf9f3137f03e4b8e532c58f698254e00f80ec"
@@ -3192,6 +3751,11 @@
resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-1.0.0.tgz#a230f64f568310e1498009940790ec99545bca7e"
integrity sha1-ojD2T1aDEOFJgAmUB5DsmVRbyn4=
+crypto-random-string@^2.0.0:
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-2.0.0.tgz#ef2a7a966ec11083388369baa02ebead229b30d5"
+ integrity sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==
+
css-animation@1.x, css-animation@^1.3.2, css-animation@^1.5.0:
version "1.6.1"
resolved "https://registry.yarnpkg.com/css-animation/-/css-animation-1.6.1.tgz#162064a3b0d51f958b7ff37b3d6d4de18e17039e"
@@ -3228,22 +3792,23 @@
postcss "^7.0.6"
postcss-selector-parser "^5.0.0-rc.4"
-css-loader@2.1.1:
- version "2.1.1"
- resolved "https://registry.yarnpkg.com/css-loader/-/css-loader-2.1.1.tgz#d8254f72e412bb2238bb44dd674ffbef497333ea"
- integrity sha512-OcKJU/lt232vl1P9EEDamhoO9iKY3tIjY5GU+XDLblAykTdgs6Ux9P1hTHve8nFKy5KPpOXOsVI/hIwi3841+w==
+css-loader@3.4.2:
+ version "3.4.2"
+ resolved "https://registry.yarnpkg.com/css-loader/-/css-loader-3.4.2.tgz#d3fdb3358b43f233b78501c5ed7b1c6da6133202"
+ integrity sha512-jYq4zdZT0oS0Iykt+fqnzVLRIeiPWhka+7BqPn+oSIpWJAHak5tmB/WZrJ2a21JhCeFyNnnlroSl8c+MtVndzA==
dependencies:
- camelcase "^5.2.0"
- icss-utils "^4.1.0"
+ camelcase "^5.3.1"
+ cssesc "^3.0.0"
+ icss-utils "^4.1.1"
loader-utils "^1.2.3"
normalize-path "^3.0.0"
- postcss "^7.0.14"
+ postcss "^7.0.23"
postcss-modules-extract-imports "^2.0.0"
- postcss-modules-local-by-default "^2.0.6"
- postcss-modules-scope "^2.1.0"
- postcss-modules-values "^2.0.0"
- postcss-value-parser "^3.3.0"
- schema-utils "^1.0.0"
+ postcss-modules-local-by-default "^3.0.2"
+ postcss-modules-scope "^2.1.1"
+ postcss-modules-values "^3.0.0"
+ postcss-value-parser "^4.0.2"
+ schema-utils "^2.6.0"
css-prefers-color-scheme@^3.1.1:
version "3.1.1"
@@ -3300,7 +3865,17 @@
resolved "https://registry.yarnpkg.com/css-what/-/css-what-3.2.1.tgz#f4a8f12421064621b456755e34a03a2c22df5da1"
integrity sha512-WwOrosiQTvyms+Ti5ZC5vGEK0Vod3FTt1ca+payZqvKuGJF+dq7bG63DstxtN0dpm6FxY27a/zS3Wten+gEtGw==
-cssdb@^4.3.0:
+css@^2.0.0:
+ version "2.2.4"
+ resolved "https://registry.yarnpkg.com/css/-/css-2.2.4.tgz#c646755c73971f2bba6a601e2cf2fd71b1298929"
+ integrity sha512-oUnjmWpy0niI3x/mPL8dVEI1l7MnG3+HHyRPHf+YFSbK+svOhXpmSOcDURUh2aOCgl2grzrOPt1nHLuCVFULLw==
+ dependencies:
+ inherits "^2.0.3"
+ source-map "^0.6.1"
+ source-map-resolve "^0.5.2"
+ urix "^0.1.0"
+
+cssdb@^4.4.0:
version "4.4.0"
resolved "https://registry.yarnpkg.com/cssdb/-/cssdb-4.4.0.tgz#3bf2f2a68c10f5c6a08abd92378331ee803cddb0"
integrity sha512-LsTAR1JPEM9TpGhl/0p3nQecC2LJ0kD8X5YARu1hk/9I1gril5vDtMZyNxcEpxxDj34YNck/ucjuoUd66K03oQ==
@@ -3373,7 +3948,7 @@
resolved "https://registry.yarnpkg.com/cssnano-util-same-parent/-/cssnano-util-same-parent-4.0.1.tgz#574082fb2859d2db433855835d9a8456ea18bbf3"
integrity sha512-WcKx5OY+KoSIAxBW6UBBRay1U6vkYheCdjyVNDm85zt5K9mHoGOfsOsqIszfAqrQQFIIKgjh2+FDgIj/zsl21Q==
-cssnano@^4.1.0:
+cssnano@^4.1.10:
version "4.1.10"
resolved "https://registry.yarnpkg.com/cssnano/-/cssnano-4.1.10.tgz#0ac41f0b13d13d465487e111b778d42da631b8b2"
integrity sha512-5wny+F6H4/8RgNlaqab4ktc3e0/blKutmq8yNlBFXA//nSFFAqAngjNVRzUvCgYROULmZZUoosL/KSoZo5aUaQ==
@@ -3407,6 +3982,13 @@
resolved "https://registry.yarnpkg.com/csstype/-/csstype-2.6.8.tgz#0fb6fc2417ffd2816a418c9336da74d7f07db431"
integrity sha512-msVS9qTuMT5zwAGCVm4mxfrZ18BNc6Csd0oJAtiFMZ1FAx1CCvy2+5MDmYoix63LM/6NDbNtodCiGYGmFgO0dA==
+currently-unhandled@^0.4.1:
+ version "0.4.1"
+ resolved "https://registry.yarnpkg.com/currently-unhandled/-/currently-unhandled-0.4.1.tgz#988df33feab191ef799a61369dd76c17adf957ea"
+ integrity sha1-mI3zP+qxke95mmE2nddsF635V+o=
+ dependencies:
+ array-find-index "^1.0.1"
+
customize-cra@^0.2.12:
version "0.2.14"
resolved "https://registry.yarnpkg.com/customize-cra/-/customize-cra-0.2.14.tgz#41f9b2d96d9a318bec760c4c9b3dc9c26d5a7594"
@@ -3419,6 +4001,14 @@
resolved "https://registry.yarnpkg.com/cyclist/-/cyclist-1.0.1.tgz#596e9698fd0c80e12038c2b82d6eb1b35b6224d9"
integrity sha1-WW6WmP0MgOEgOMK4LW6xs1tiJNk=
+d@1, d@^1.0.1:
+ version "1.0.1"
+ resolved "https://registry.yarnpkg.com/d/-/d-1.0.1.tgz#8698095372d58dbee346ffd0c7093f99f8f9eb5a"
+ integrity sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==
+ dependencies:
+ es5-ext "^0.10.50"
+ type "^1.0.1"
+
damerau-levenshtein@^1.0.4:
version "1.0.5"
resolved "https://registry.yarnpkg.com/damerau-levenshtein/-/damerau-levenshtein-1.0.5.tgz#780cf7144eb2e8dbd1c3bb83ae31100ccc31a414"
@@ -3468,18 +4058,19 @@
dependencies:
ms "^2.1.1"
-decamelize@^1.2.0:
+decamelize-keys@^1.0.0:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/decamelize-keys/-/decamelize-keys-1.1.0.tgz#d171a87933252807eb3cb61dc1c1445d078df2d9"
+ integrity sha1-0XGoeTMlKAfrPLYdwcFEXQeN8tk=
+ dependencies:
+ decamelize "^1.1.0"
+ map-obj "^1.0.0"
+
+decamelize@^1.1.0, decamelize@^1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290"
integrity sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=
-decamelize@^2.0.0:
- version "2.0.0"
- resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-2.0.0.tgz#656d7bbc8094c4c788ea53c5840908c9c7d063c7"
- integrity sha512-Ikpp5scV3MSYxY39ymh45ZLEecsTdv/Xj2CaQfI8RLMuwi7XvjX9H/fhraiSuU+C5w5NTDu4ZU72xNiZnurBPg==
- dependencies:
- xregexp "4.0.0"
-
decode-uri-component@^0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/decode-uri-component/-/decode-uri-component-0.2.0.tgz#eb3913333458775cb84cd1a1fae062106bb87545"
@@ -3514,6 +4105,13 @@
resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34"
integrity sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=
+deep-strict-equal@^0.2.0:
+ version "0.2.0"
+ resolved "https://registry.yarnpkg.com/deep-strict-equal/-/deep-strict-equal-0.2.0.tgz#4a078147a8ab57f6a0d4f5547243cd22f44eb4e4"
+ integrity sha1-SgeBR6irV/ag1PVUckPNIvROtOQ=
+ dependencies:
+ core-assert "^0.2.0"
+
default-gateway@^4.2.0:
version "4.2.0"
resolved "https://registry.yarnpkg.com/default-gateway/-/default-gateway-4.2.0.tgz#167104c7500c2115f6dd69b0a536bb8ed720552b"
@@ -3556,17 +4154,18 @@
is-descriptor "^1.0.2"
isobject "^3.0.1"
-del@^3.0.0:
- version "3.0.0"
- resolved "https://registry.yarnpkg.com/del/-/del-3.0.0.tgz#53ecf699ffcbcb39637691ab13baf160819766e5"
- integrity sha1-U+z2mf/LyzljdpGrE7rxYIGXZuU=
+del@^4.1.1:
+ version "4.1.1"
+ resolved "https://registry.yarnpkg.com/del/-/del-4.1.1.tgz#9e8f117222ea44a31ff3a156c049b99052a9f0b4"
+ integrity sha512-QwGuEUouP2kVwQenAsOof5Fv8K9t3D8Ca8NxcXKrIpEHjTXK5J2nXLdP+ALI1cgv8wj7KuwBhTwBkOZSJKM5XQ==
dependencies:
+ "@types/glob" "^7.1.1"
globby "^6.1.0"
- is-path-cwd "^1.0.0"
- is-path-in-cwd "^1.0.0"
- p-map "^1.1.1"
- pify "^3.0.0"
- rimraf "^2.2.8"
+ is-path-cwd "^2.0.0"
+ is-path-in-cwd "^2.0.0"
+ p-map "^2.0.0"
+ pify "^4.0.1"
+ rimraf "^2.6.3"
delayed-stream@~1.0.0:
version "1.0.0"
@@ -3631,6 +4230,13 @@
arrify "^1.0.1"
path-type "^3.0.0"
+dir-glob@^2.2.2:
+ version "2.2.2"
+ resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-2.2.2.tgz#fa09f0694153c8918b18ba0deafae94769fc50c4"
+ integrity sha512-f9LBi5QWzIW3I6e//uxZoLBlUt9kcp66qo0sSCxL6YZKc75R1c4MFCoe/LaZiBGmgujvQdxc5Bn3QhfyvK5Hsw==
+ dependencies:
+ path-type "^3.0.0"
+
dns-equal@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/dns-equal/-/dns-equal-1.0.0.tgz#b39e7f1da6eb0a75ba9c17324b34753c47e0654d"
@@ -3755,6 +4361,14 @@
dom-serializer "0"
domelementtype "1"
+dot-case@^3.0.3:
+ version "3.0.3"
+ resolved "https://registry.yarnpkg.com/dot-case/-/dot-case-3.0.3.tgz#21d3b52efaaba2ea5fda875bb1aa8124521cf4aa"
+ integrity sha512-7hwEmg6RiSQfm/GwPL4AAWXKy3YNNZA3oFv2Pdiey0mwkRCPZ9x6SZbkLcn8Ma5PYeVokzoD4Twv2n7LKp5WeA==
+ dependencies:
+ no-case "^3.0.3"
+ tslib "^1.10.0"
+
dot-prop@^4.1.0, dot-prop@^4.1.1:
version "4.2.0"
resolved "https://registry.yarnpkg.com/dot-prop/-/dot-prop-4.2.0.tgz#1f19e0c2e1aa0e32797c49799f2837ac6af69c57"
@@ -3762,15 +4376,22 @@
dependencies:
is-obj "^1.0.0"
-dotenv-expand@4.2.0:
- version "4.2.0"
- resolved "https://registry.yarnpkg.com/dotenv-expand/-/dotenv-expand-4.2.0.tgz#def1f1ca5d6059d24a766e587942c21106ce1275"
- integrity sha1-3vHxyl1gWdJKdm5YeULCEQbOEnU=
+dot-prop@^5.2.0:
+ version "5.2.0"
+ resolved "https://registry.yarnpkg.com/dot-prop/-/dot-prop-5.2.0.tgz#c34ecc29556dc45f1f4c22697b6f4904e0cc4fcb"
+ integrity sha512-uEUyaDKoSQ1M4Oq8l45hSE26SnTxL6snNnqvK/VWx5wJhmff5z0FUVJDKDanor/6w3kzE3i7XZOk+7wC0EXr1A==
+ dependencies:
+ is-obj "^2.0.0"
-dotenv@6.2.0:
- version "6.2.0"
- resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-6.2.0.tgz#941c0410535d942c8becf28d3f357dbd9d476064"
- integrity sha512-HygQCKUBSFl8wKQZBSemMywRWcEDNidvNbjGVyZu3nbZ8qq9ubiPoGLMdRDpfSrpkkm9BXYFkpKxxFX38o/76w==
+dotenv-expand@5.1.0:
+ version "5.1.0"
+ resolved "https://registry.yarnpkg.com/dotenv-expand/-/dotenv-expand-5.1.0.tgz#3fbaf020bfd794884072ea26b1e9791d45a629f0"
+ integrity sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA==
+
+dotenv@8.2.0:
+ version "8.2.0"
+ resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.2.0.tgz#97e619259ada750eea3e4ea3e26bceea5424b16a"
+ integrity sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw==
draft-js@^0.10.0, draft-js@~0.10.0:
version "0.10.5"
@@ -3814,11 +4435,16 @@
resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d"
integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=
-electron-to-chromium@^1.3.247, electron-to-chromium@^1.3.322:
+electron-to-chromium@^1.3.322:
version "1.3.328"
resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.328.tgz#a619575c42f1d5b443103664f25ffa5a80190ee5"
integrity sha512-x4XefnFxDxFwaQ01d/pppJP9meWhOIJ/gtI6/4jqkpsadq79uL7NYSaX64naLmJqvzUBjSrO3IM2+1b/W9KdPg==
+electron-to-chromium@^1.3.378, electron-to-chromium@^1.3.413:
+ version "1.3.431"
+ resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.431.tgz#705dd8ef46200415ba837b31d927cdc1e43db303"
+ integrity sha512-2okqkXCIda7qDwjYhUFxPcQdZDIZZ/zBLDzVOif7WW/TSNfEhdT6SO07O1x/sFteEHX189Z//UwjbZKKCOn2Fg==
+
elliptic@^6.0.0:
version "6.5.2"
resolved "https://registry.yarnpkg.com/elliptic/-/elliptic-6.5.2.tgz#05c5678d7173c049d8ca433552224a495d0e3762"
@@ -3837,11 +4463,21 @@
resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-7.0.3.tgz#933a04052860c85e83c122479c4748a8e4c72156"
integrity sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==
+emoji-regex@^8.0.0:
+ version "8.0.0"
+ resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37"
+ integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==
+
emojis-list@^2.0.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/emojis-list/-/emojis-list-2.1.0.tgz#4daa4d9db00f9819880c79fa457ae5b09a1fd389"
integrity sha1-TapNnbAPmBmIDHn6RXrlsJof04k=
+emojis-list@^3.0.0:
+ version "3.0.0"
+ resolved "https://registry.yarnpkg.com/emojis-list/-/emojis-list-3.0.0.tgz#5570662046ad29e2e916e71aae260abdff4f6a78"
+ integrity sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q==
+
encodeurl@~1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59"
@@ -3861,6 +4497,22 @@
dependencies:
once "^1.4.0"
+enhance-visitors@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/enhance-visitors/-/enhance-visitors-1.0.0.tgz#aa945d05da465672a1ebd38fee2ed3da8518e95a"
+ integrity sha1-qpRdBdpGVnKh69OP7i7T2oUY6Vo=
+ dependencies:
+ lodash "^4.13.1"
+
+enhanced-resolve@^0.9.1:
+ version "0.9.1"
+ resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-0.9.1.tgz#4d6e689b3725f86090927ccc86cd9f1635b89e2e"
+ integrity sha1-TW5omzcl+GCQknzMhs2fFjW4ni4=
+ dependencies:
+ graceful-fs "^4.1.2"
+ memory-fs "^0.2.0"
+ tapable "^0.1.8"
+
enhanced-resolve@^4.1.0:
version "4.1.1"
resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-4.1.1.tgz#2937e2b8066cd0fe7ce0990a98f0d71a35189f66"
@@ -3885,6 +4537,11 @@
resolved "https://registry.yarnpkg.com/entities/-/entities-2.0.0.tgz#68d6084cab1b079767540d80e56a39b423e4abf4"
integrity sha512-D9f7V0JSRwIxlRI2mjMqufDrRDnx8p+eEOz7aUM9SuvF8gsBzra0/6tbjl1m8eQHrZlYj6PxqE00hZ1SAIKPLw==
+env-editor@^0.4.0:
+ version "0.4.1"
+ resolved "https://registry.yarnpkg.com/env-editor/-/env-editor-0.4.1.tgz#77011e08ce45f46e404e8d996b465c684ca57502"
+ integrity sha512-suh+Vm00GnPQgXpmONTkcUT9LgBSL6sJrRnJxbykT0j+ONjzmIS+1U3ne467ArdZN/42/npp+GnhtwkLQ+vUjw==
+
errno@^0.1.1, errno@^0.1.3, errno@~0.1.7:
version "0.1.7"
resolved "https://registry.yarnpkg.com/errno/-/errno-0.1.7.tgz#4684d71779ad39af177e3f007996f7c67c852618"
@@ -3933,12 +4590,48 @@
is-date-object "^1.0.1"
is-symbol "^1.0.2"
+es5-ext@^0.10.35, es5-ext@^0.10.50:
+ version "0.10.53"
+ resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.10.53.tgz#93c5a3acfdbef275220ad72644ad02ee18368de1"
+ integrity sha512-Xs2Stw6NiNHWypzRTY1MtaG/uJlwCk8kH81920ma8mvN8Xq1gsfhZvpkImLQArw8AHnv8MT2I45J3c0R8slE+Q==
+ dependencies:
+ es6-iterator "~2.0.3"
+ es6-symbol "~3.1.3"
+ next-tick "~1.0.0"
+
+es6-iterator@2.0.3, es6-iterator@~2.0.3:
+ version "2.0.3"
+ resolved "https://registry.yarnpkg.com/es6-iterator/-/es6-iterator-2.0.3.tgz#a7de889141a05a94b0854403b2d0a0fbfa98f3b7"
+ integrity sha1-p96IkUGgWpSwhUQDstCg+/qY87c=
+ dependencies:
+ d "1"
+ es5-ext "^0.10.35"
+ es6-symbol "^3.1.1"
+
+es6-symbol@^3.1.1, es6-symbol@~3.1.3:
+ version "3.1.3"
+ resolved "https://registry.yarnpkg.com/es6-symbol/-/es6-symbol-3.1.3.tgz#bad5d3c1bcdac28269f4cb331e431c78ac705d18"
+ integrity sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==
+ dependencies:
+ d "^1.0.1"
+ ext "^1.1.2"
+
+escape-goat@^2.0.0:
+ version "2.1.1"
+ resolved "https://registry.yarnpkg.com/escape-goat/-/escape-goat-2.1.1.tgz#1b2dc77003676c457ec760b2dc68edb648188675"
+ integrity sha512-8/uIhbG12Csjy2JEW7D9pHbreaVaS/OpN3ycnyvElTdwM5n6GY6W6e2IPemfvGZeUMqZ9A/3GqIZMgKnBhAw/Q==
+
escape-html@~1.0.3:
version "1.0.3"
resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988"
integrity sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=
-escape-string-regexp@1.0.5, escape-string-regexp@^1.0.2, escape-string-regexp@^1.0.5:
+escape-string-regexp@2.0.0:
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz#a30304e99daa32e23b2fd20f51babd07cffca344"
+ integrity sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==
+
+escape-string-regexp@^1.0.2, escape-string-regexp@^1.0.5:
version "1.0.5"
resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4"
integrity sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=
@@ -3955,12 +4648,57 @@
optionalDependencies:
source-map "~0.6.1"
-eslint-config-react-app@^4.0.0:
- version "4.0.1"
- resolved "https://registry.yarnpkg.com/eslint-config-react-app/-/eslint-config-react-app-4.0.1.tgz#23fd0fd7ea89442ef1e733f66a7207674b23c8db"
- integrity sha512-ZsaoXUIGsK8FCi/x4lT2bZR5mMkL/Kgj+Lnw690rbvvUr/uiwgFiD8FcfAhkCycm7Xte6O5lYz4EqMx2vX7jgw==
+eslint-ast-utils@^1.1.0:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/eslint-ast-utils/-/eslint-ast-utils-1.1.0.tgz#3d58ba557801cfb1c941d68131ee9f8c34bd1586"
+ integrity sha512-otzzTim2/1+lVrlH19EfQQJEhVJSu0zOb9ygb3iapN6UlyaDtyRq4b5U1FuW0v1lRa9Fp/GJyHkSwm6NqABgCA==
dependencies:
- confusing-browser-globals "^1.0.7"
+ lodash.get "^4.4.2"
+ lodash.zip "^4.2.0"
+
+eslint-config-prettier@^6.11.0:
+ version "6.11.0"
+ resolved "https://registry.yarnpkg.com/eslint-config-prettier/-/eslint-config-prettier-6.11.0.tgz#f6d2238c1290d01c859a8b5c1f7d352a0b0da8b1"
+ integrity sha512-oB8cpLWSAjOVFEJhhyMZh6NOEOtBVziaqdDQ86+qhDHFbZXoRTM7pNSvFRfW/W/L/LrQ38C99J5CGuRBBzBsdA==
+ dependencies:
+ get-stdin "^6.0.0"
+
+eslint-config-react-app@^5.2.1:
+ version "5.2.1"
+ resolved "https://registry.yarnpkg.com/eslint-config-react-app/-/eslint-config-react-app-5.2.1.tgz#698bf7aeee27f0cea0139eaef261c7bf7dd623df"
+ integrity sha512-pGIZ8t0mFLcV+6ZirRgYK6RVqUIKRIi9MmgzUEmrIknsn3AdO0I32asO86dJgloHq+9ZPl8UIg8mYrvgP5u2wQ==
+ dependencies:
+ confusing-browser-globals "^1.0.9"
+
+eslint-config-xo-react@^0.23.0:
+ version "0.23.0"
+ resolved "https://registry.yarnpkg.com/eslint-config-xo-react/-/eslint-config-xo-react-0.23.0.tgz#a6e4cfc3e1231cf9291fa19000f559dc53183090"
+ integrity sha512-yBNoMF6COjJLRijlWksC1Jws6CGWHMgTfR/PAReCdZUo5cg/ZsyNeZx/fntyU4SbARwW2RvuezMeKle+jI5yoA==
+
+eslint-config-xo-typescript@^0.28.0:
+ version "0.28.0"
+ resolved "https://registry.yarnpkg.com/eslint-config-xo-typescript/-/eslint-config-xo-typescript-0.28.0.tgz#312286a90881097865f73e3237081a12e90f16c4"
+ integrity sha512-q+mBhTikLjrPOszx6gezd6ZxrbrSHeBp/Z8gGgPowBYvBYG78llfQZxOcsbKN6aM8+S/OFAJP9bk5w/roLDFTA==
+
+eslint-config-xo@^0.29.0, eslint-config-xo@^0.29.1:
+ version "0.29.1"
+ resolved "https://registry.yarnpkg.com/eslint-config-xo/-/eslint-config-xo-0.29.1.tgz#876e29b2f4711f2fd365885b09b9536b6ef328dc"
+ integrity sha512-RDjeKh8CV0/EH4utW/6uOkwJJOOU+rX3uE5eUBOamcLNe4lNjyo8kSt3B6DzAm1L/1tWGikI7LFNVY9gG7PDQw==
+ dependencies:
+ confusing-browser-globals "1.0.9"
+
+eslint-formatter-pretty@^3.0.1:
+ version "3.0.1"
+ resolved "https://registry.yarnpkg.com/eslint-formatter-pretty/-/eslint-formatter-pretty-3.0.1.tgz#97603fcb2ddcc6dd60662d6e9f327a734cc55a54"
+ integrity sha512-hhQ/ASD4i6BAEalcEfUxesFtJFftT8xFsimCzUpPbTzygJ4J17yCGcJ3XKCB2g7XTJTv0pi7rVTadfHVmtfSRA==
+ dependencies:
+ ansi-escapes "^4.2.1"
+ chalk "^3.0.0"
+ eslint-rule-docs "^1.1.5"
+ log-symbols "^3.0.0"
+ plur "^3.0.1"
+ string-width "^4.2.0"
+ supports-hyperlinks "^2.0.0"
eslint-import-resolver-node@^0.3.2:
version "0.3.2"
@@ -3970,53 +4708,120 @@
debug "^2.6.9"
resolve "^1.5.0"
-eslint-loader@2.1.2:
- version "2.1.2"
- resolved "https://registry.yarnpkg.com/eslint-loader/-/eslint-loader-2.1.2.tgz#453542a1230d6ffac90e4e7cb9cadba9d851be68"
- integrity sha512-rA9XiXEOilLYPOIInvVH5S/hYfyTPyxag6DZhoQOduM+3TkghAEQ3VcFO8VnX4J4qg/UIBzp72aOf/xvYmpmsg==
+eslint-import-resolver-webpack@^0.12.1:
+ version "0.12.1"
+ resolved "https://registry.yarnpkg.com/eslint-import-resolver-webpack/-/eslint-import-resolver-webpack-0.12.1.tgz#771ae561e887ca4e53ee87605fbb36c5e290b0f5"
+ integrity sha512-O/sUAXk6GWrICiN8JUkkjdt9uZpqZHP+FVnTxtEILL6EZMaPSrnP4lGPSFwcKsv7O211maqq4Nz60+dh236hVg==
dependencies:
- loader-fs-cache "^1.0.0"
- loader-utils "^1.0.2"
- object-assign "^4.0.1"
- object-hash "^1.1.4"
- rimraf "^2.6.1"
+ array-find "^1.0.0"
+ debug "^2.6.9"
+ enhanced-resolve "^0.9.1"
+ find-root "^1.1.0"
+ has "^1.0.3"
+ interpret "^1.2.0"
+ lodash "^4.17.15"
+ node-libs-browser "^1.0.0 || ^2.0.0"
+ resolve "^1.13.1"
+ semver "^5.7.1"
-eslint-module-utils@^2.3.0:
- version "2.5.0"
- resolved "https://registry.yarnpkg.com/eslint-module-utils/-/eslint-module-utils-2.5.0.tgz#cdf0b40d623032274ccd2abd7e64c4e524d6e19c"
- integrity sha512-kCo8pZaNz2dsAW7nCUjuVoI11EBXXpIzfNxmaoLhXoRDOnqXLC4iSGVRdZPhOitfbdEfMEfKOiENaK6wDPZEGw==
+eslint-loader@3.0.3:
+ version "3.0.3"
+ resolved "https://registry.yarnpkg.com/eslint-loader/-/eslint-loader-3.0.3.tgz#e018e3d2722381d982b1201adb56819c73b480ca"
+ integrity sha512-+YRqB95PnNvxNp1HEjQmvf9KNvCin5HXYYseOXVC2U0KEcw4IkQ2IQEBG46j7+gW39bMzeu0GsUhVbBY3Votpw==
+ dependencies:
+ fs-extra "^8.1.0"
+ loader-fs-cache "^1.0.2"
+ loader-utils "^1.2.3"
+ object-hash "^2.0.1"
+ schema-utils "^2.6.1"
+
+eslint-module-utils@^2.4.1:
+ version "2.6.0"
+ resolved "https://registry.yarnpkg.com/eslint-module-utils/-/eslint-module-utils-2.6.0.tgz#579ebd094f56af7797d19c9866c9c9486629bfa6"
+ integrity sha512-6j9xxegbqe8/kZY8cYpcp0xhbK0EgJlg3g9mib3/miLaExuuwc3n5UEfSnU6hWMbT0FAYVvDbL9RrRgpUeQIvA==
dependencies:
debug "^2.6.9"
pkg-dir "^2.0.0"
-eslint-plugin-flowtype@2.50.1:
- version "2.50.1"
- resolved "https://registry.yarnpkg.com/eslint-plugin-flowtype/-/eslint-plugin-flowtype-2.50.1.tgz#36d4c961ac8b9e9e1dc091d3fba0537dad34ae8a"
- integrity sha512-9kRxF9hfM/O6WGZcZPszOVPd2W0TLHBtceulLTsGfwMPtiCCLnCW0ssRiOOiXyqrCA20pm1iXdXm7gQeN306zQ==
+eslint-plugin-ava@^10.0.1:
+ version "10.3.0"
+ resolved "https://registry.yarnpkg.com/eslint-plugin-ava/-/eslint-plugin-ava-10.3.0.tgz#8c388520ae95bc4681d06f25fbcd0d70f67163e1"
+ integrity sha512-rNijYwBo+28ahXvifTx3IYzsNjtArAeyowBJ1Rd5dCtW7f0y5aSTBU7sXBpU5o5Eh5/FZvz3lSLVLac8ExYTtA==
dependencies:
- lodash "^4.17.10"
+ deep-strict-equal "^0.2.0"
+ enhance-visitors "^1.0.0"
+ espree "^6.1.2"
+ espurify "^2.0.1"
+ import-modules "^2.0.0"
+ micro-spelling-correcter "^1.1.1"
+ pkg-dir "^4.2.0"
+ resolve-from "^5.0.0"
-eslint-plugin-import@2.16.0:
- version "2.16.0"
- resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.16.0.tgz#97ac3e75d0791c4fac0e15ef388510217be7f66f"
- integrity sha512-z6oqWlf1x5GkHIFgrSvtmudnqM6Q60KM4KvpWi5ubonMjycLjndvd5+8VAZIsTlHC03djdgJuyKG6XO577px6A==
+eslint-plugin-es@^3.0.0:
+ version "3.0.0"
+ resolved "https://registry.yarnpkg.com/eslint-plugin-es/-/eslint-plugin-es-3.0.0.tgz#98cb1bc8ab0aa807977855e11ad9d1c9422d014b"
+ integrity sha512-6/Jb/J/ZvSebydwbBJO1R9E5ky7YeElfK56Veh7e4QGFHCXoIXGH9HhVz+ibJLM3XJ1XjP+T7rKBLUa/Y7eIng==
dependencies:
+ eslint-utils "^2.0.0"
+ regexpp "^3.0.0"
+
+eslint-plugin-eslint-comments@^3.1.2:
+ version "3.1.2"
+ resolved "https://registry.yarnpkg.com/eslint-plugin-eslint-comments/-/eslint-plugin-eslint-comments-3.1.2.tgz#4ef6c488dbe06aa1627fea107b3e5d059fc8a395"
+ integrity sha512-QexaqrNeteFfRTad96W+Vi4Zj1KFbkHHNMMaHZEYcovKav6gdomyGzaxSDSL3GoIyUOo078wRAdYlu1caiauIQ==
+ dependencies:
+ escape-string-regexp "^1.0.5"
+ ignore "^5.0.5"
+
+eslint-plugin-flowtype@4.6.0:
+ version "4.6.0"
+ resolved "https://registry.yarnpkg.com/eslint-plugin-flowtype/-/eslint-plugin-flowtype-4.6.0.tgz#82b2bd6f21770e0e5deede0228e456cb35308451"
+ integrity sha512-W5hLjpFfZyZsXfo5anlu7HM970JBDqbEshAJUkeczP6BFCIfJXuiIBQXyberLRtOStT0OGPF8efeTbxlHk4LpQ==
+ dependencies:
+ lodash "^4.17.15"
+
+eslint-plugin-import@2.20.1:
+ version "2.20.1"
+ resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.20.1.tgz#802423196dcb11d9ce8435a5fc02a6d3b46939b3"
+ integrity sha512-qQHgFOTjguR+LnYRoToeZWT62XM55MBVXObHM6SKFd1VzDcX/vqT1kAz8ssqigh5eMj8qXcRoXXGZpPP6RfdCw==
+ dependencies:
+ array-includes "^3.0.3"
+ array.prototype.flat "^1.2.1"
contains-path "^0.1.0"
debug "^2.6.9"
doctrine "1.5.0"
eslint-import-resolver-node "^0.3.2"
- eslint-module-utils "^2.3.0"
+ eslint-module-utils "^2.4.1"
has "^1.0.3"
- lodash "^4.17.11"
minimatch "^3.0.4"
+ object.values "^1.1.0"
read-pkg-up "^2.0.0"
- resolve "^1.9.0"
+ resolve "^1.12.0"
-eslint-plugin-jsx-a11y@6.2.1:
- version "6.2.1"
- resolved "https://registry.yarnpkg.com/eslint-plugin-jsx-a11y/-/eslint-plugin-jsx-a11y-6.2.1.tgz#4ebba9f339b600ff415ae4166e3e2e008831cf0c"
- integrity sha512-cjN2ObWrRz0TTw7vEcGQrx+YltMvZoOEx4hWU8eEERDnBIU00OTq7Vr+jA7DFKxiwLNv4tTh5Pq2GUNEa8b6+w==
+eslint-plugin-import@^2.20.1, eslint-plugin-import@^2.20.2:
+ version "2.20.2"
+ resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.20.2.tgz#91fc3807ce08be4837141272c8b99073906e588d"
+ integrity sha512-FObidqpXrR8OnCh4iNsxy+WACztJLXAHBO5hK79T1Hc77PgQZkyDGA5Ag9xAvRpglvLNxhH/zSmZ70/pZ31dHg==
dependencies:
+ array-includes "^3.0.3"
+ array.prototype.flat "^1.2.1"
+ contains-path "^0.1.0"
+ debug "^2.6.9"
+ doctrine "1.5.0"
+ eslint-import-resolver-node "^0.3.2"
+ eslint-module-utils "^2.4.1"
+ has "^1.0.3"
+ minimatch "^3.0.4"
+ object.values "^1.1.0"
+ read-pkg-up "^2.0.0"
+ resolve "^1.12.0"
+
+eslint-plugin-jsx-a11y@6.2.3:
+ version "6.2.3"
+ resolved "https://registry.yarnpkg.com/eslint-plugin-jsx-a11y/-/eslint-plugin-jsx-a11y-6.2.3.tgz#b872a09d5de51af70a97db1eea7dc933043708aa"
+ integrity sha512-CawzfGt9w83tyuVekn0GDPU9ytYtxyxyFZ3aSWROmnRRFQFT2BiPJd7jvRdzNDi6oLWaS2asMeYSNMjWTV4eNg==
+ dependencies:
+ "@babel/runtime" "^7.4.5"
aria-query "^3.0.0"
array-includes "^3.0.3"
ast-types-flow "^0.0.7"
@@ -4024,35 +4829,89 @@
damerau-levenshtein "^1.0.4"
emoji-regex "^7.0.2"
has "^1.0.3"
- jsx-ast-utils "^2.0.1"
+ jsx-ast-utils "^2.2.1"
-eslint-plugin-react-hooks@^1.5.0:
+eslint-plugin-no-use-extend-native@^0.5.0:
+ version "0.5.0"
+ resolved "https://registry.yarnpkg.com/eslint-plugin-no-use-extend-native/-/eslint-plugin-no-use-extend-native-0.5.0.tgz#d6855e3a823a819b467cf7df56adca57de741bf9"
+ integrity sha512-dBNjs8hor8rJgeXLH4HTut5eD3RGWf9JUsadIfuL7UosVQ/dnvOKwxEcRrXrFxrMZ8llUVWT+hOimxJABsAUzQ==
+ dependencies:
+ is-get-set-prop "^1.0.0"
+ is-js-type "^2.0.0"
+ is-obj-prop "^1.0.0"
+ is-proto-prop "^2.0.0"
+
+eslint-plugin-node@^11.0.0:
+ version "11.1.0"
+ resolved "https://registry.yarnpkg.com/eslint-plugin-node/-/eslint-plugin-node-11.1.0.tgz#c95544416ee4ada26740a30474eefc5402dc671d"
+ integrity sha512-oUwtPJ1W0SKD0Tr+wqu92c5xuCeQqB3hSCHasn/ZgjFdA9iDGNkNf2Zi9ztY7X+hNuMib23LNGRm6+uN+KLE3g==
+ dependencies:
+ eslint-plugin-es "^3.0.0"
+ eslint-utils "^2.0.0"
+ ignore "^5.1.1"
+ minimatch "^3.0.4"
+ resolve "^1.10.1"
+ semver "^6.1.0"
+
+eslint-plugin-prettier@^3.1.3:
+ version "3.1.3"
+ resolved "https://registry.yarnpkg.com/eslint-plugin-prettier/-/eslint-plugin-prettier-3.1.3.tgz#ae116a0fc0e598fdae48743a4430903de5b4e6ca"
+ integrity sha512-+HG5jmu/dN3ZV3T6eCD7a4BlAySdN7mLIbJYo0z1cFQuI+r2DiTJEFeF68ots93PsnrMxbzIZ2S/ieX+mkrBeQ==
+ dependencies:
+ prettier-linter-helpers "^1.0.0"
+
+eslint-plugin-promise@^4.2.1:
+ version "4.2.1"
+ resolved "https://registry.yarnpkg.com/eslint-plugin-promise/-/eslint-plugin-promise-4.2.1.tgz#845fd8b2260ad8f82564c1222fce44ad71d9418a"
+ integrity sha512-VoM09vT7bfA7D+upt+FjeBO5eHIJQBUWki1aPvB+vbNiHS3+oGIJGIeyBtKQTME6UPXXy3vV07OL1tHd3ANuDw==
+
+eslint-plugin-react-hooks@^1.6.1:
version "1.7.0"
resolved "https://registry.yarnpkg.com/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-1.7.0.tgz#6210b6d5a37205f0b92858f895a4e827020a7d04"
integrity sha512-iXTCFcOmlWvw4+TOE8CLWj6yX1GwzT0Y6cUfHHZqWnSk144VmVIRcVGtUAzrLES7C798lmvnt02C7rxaOX1HNA==
-eslint-plugin-react@7.12.4:
- version "7.12.4"
- resolved "https://registry.yarnpkg.com/eslint-plugin-react/-/eslint-plugin-react-7.12.4.tgz#b1ecf26479d61aee650da612e425c53a99f48c8c"
- integrity sha512-1puHJkXJY+oS1t467MjbqjvX53uQ05HXwjqDgdbGBqf5j9eeydI54G3KwiJmWciQ0HTBacIKw2jgwSBSH3yfgQ==
+eslint-plugin-react@7.19.0, eslint-plugin-react@^7.19.0:
+ version "7.19.0"
+ resolved "https://registry.yarnpkg.com/eslint-plugin-react/-/eslint-plugin-react-7.19.0.tgz#6d08f9673628aa69c5559d33489e855d83551666"
+ integrity sha512-SPT8j72CGuAP+JFbT0sJHOB80TX/pu44gQ4vXH/cq+hQTiY2PuZ6IHkqXJV6x1b28GDdo1lbInjKUrrdUf0LOQ==
dependencies:
- array-includes "^3.0.3"
+ array-includes "^3.1.1"
doctrine "^2.1.0"
has "^1.0.3"
- jsx-ast-utils "^2.0.1"
- object.fromentries "^2.0.0"
- prop-types "^15.6.2"
- resolve "^1.9.0"
+ jsx-ast-utils "^2.2.3"
+ object.entries "^1.1.1"
+ object.fromentries "^2.0.2"
+ object.values "^1.1.1"
+ prop-types "^15.7.2"
+ resolve "^1.15.1"
+ semver "^6.3.0"
+ string.prototype.matchall "^4.0.2"
+ xregexp "^4.3.0"
-eslint-scope@3.7.1:
- version "3.7.1"
- resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-3.7.1.tgz#3d63c3edfda02e06e01a452ad88caacc7cdcb6e8"
- integrity sha1-PWPD7f2gLgbgGkUq2IyqzHzctug=
+eslint-plugin-unicorn@^19.0.0, eslint-plugin-unicorn@^19.0.1:
+ version "19.0.1"
+ resolved "https://registry.yarnpkg.com/eslint-plugin-unicorn/-/eslint-plugin-unicorn-19.0.1.tgz#05eef02f33426b8aa4f21cd5e4785b456335b85b"
+ integrity sha512-fu0/h5mHXfBC6EkA3i2vCjsfC8j53+T9txGhNL4fpxJ+1JKsUKfv+tmXDgy0XnLHhFjnOZp4tRWJWbcykeIP2Q==
dependencies:
- esrecurse "^4.1.0"
- estraverse "^4.1.1"
+ ci-info "^2.0.0"
+ clean-regexp "^1.0.0"
+ eslint-ast-utils "^1.1.0"
+ eslint-template-visitor "^1.1.0"
+ eslint-utils "^2.0.0"
+ import-modules "^2.0.0"
+ lodash "^4.17.15"
+ read-pkg-up "^7.0.1"
+ regexp-tree "^0.1.21"
+ reserved-words "^0.1.2"
+ safe-regex "^2.1.1"
+ semver "^7.1.3"
-eslint-scope@^4.0.0, eslint-scope@^4.0.3:
+eslint-rule-docs@^1.1.5:
+ version "1.1.189"
+ resolved "https://registry.yarnpkg.com/eslint-rule-docs/-/eslint-rule-docs-1.1.189.tgz#56b4bdf223774dc66c412365e6ba0017d297a9c5"
+ integrity sha512-26nV3Ef6J9xUjQhTjnUlB628AEXuQcRvXzUSQz04tTm1TSIUyBe2XaI8ODDxK68/42QifDBbdNuMFBtcHNDmGQ==
+
+eslint-scope@^4.0.3:
version "4.0.3"
resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-4.0.3.tgz#ca03833310f6889a3264781aa82e63eb9cfe7848"
integrity sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg==
@@ -4060,68 +4919,93 @@
esrecurse "^4.1.0"
estraverse "^4.1.1"
-eslint-utils@^1.3.1:
+eslint-scope@^5.0.0:
+ version "5.0.0"
+ resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-5.0.0.tgz#e87c8887c73e8d1ec84f1ca591645c358bfc8fb9"
+ integrity sha512-oYrhJW7S0bxAFDvWqzvMPRm6pcgcnWc4QnofCAqRTRfQC0JcwenzGglTtsLyIuuWFfkqDG9vz67cnttSd53djw==
+ dependencies:
+ esrecurse "^4.1.0"
+ estraverse "^4.1.1"
+
+eslint-template-visitor@^1.1.0:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/eslint-template-visitor/-/eslint-template-visitor-1.1.0.tgz#f090d124d1a52e05552149fc50468ed59608b166"
+ integrity sha512-Lmy6QVlmFiIGl5fPi+8ACnov3sare+0Ouf7deJAGGhmUfeWJ5fVarELUxZRpsZ9sHejiJUq8626d0dn9uvcZTw==
+ dependencies:
+ eslint-visitor-keys "^1.1.0"
+ espree "^6.1.1"
+ multimap "^1.0.2"
+
+eslint-utils@^1.4.3:
version "1.4.3"
resolved "https://registry.yarnpkg.com/eslint-utils/-/eslint-utils-1.4.3.tgz#74fec7c54d0776b6f67e0251040b5806564e981f"
integrity sha512-fbBN5W2xdY45KulGXmLHZ3c3FHfVYmKg0IrAKGOkT/464PQsx2UeIzfz1RmEci+KLm1bBaAzZAh8+/E+XAeZ8Q==
dependencies:
eslint-visitor-keys "^1.1.0"
+eslint-utils@^2.0.0:
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/eslint-utils/-/eslint-utils-2.0.0.tgz#7be1cc70f27a72a76cd14aa698bcabed6890e1cd"
+ integrity sha512-0HCPuJv+7Wv1bACm8y5/ECVfYdfsAm9xmVb7saeFlxjPYALefjhbYoCkBjPdPzGH8wWyTpAez82Fh3VKYEZ8OA==
+ dependencies:
+ eslint-visitor-keys "^1.1.0"
+
eslint-visitor-keys@^1.0.0, eslint-visitor-keys@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-1.1.0.tgz#e2a82cea84ff246ad6fb57f9bde5b46621459ec2"
integrity sha512-8y9YjtM1JBJU/A9Kc+SbaOV4y29sSWckBwMHa+FGtVj5gN/sbnKDf6xJUl+8g7FAij9LVaP8C24DUiH/f/2Z9A==
-eslint@^5.16.0:
- version "5.16.0"
- resolved "https://registry.yarnpkg.com/eslint/-/eslint-5.16.0.tgz#a1e3ac1aae4a3fbd8296fcf8f7ab7314cbb6abea"
- integrity sha512-S3Rz11i7c8AA5JPv7xAH+dOyq/Cu/VXHiHXBPOU1k/JAM5dXqQPt3qcrhpHSorXmrpu2g0gkIBVXAqCpzfoZIg==
+eslint@^6.6.0, eslint@^6.8.0:
+ version "6.8.0"
+ resolved "https://registry.yarnpkg.com/eslint/-/eslint-6.8.0.tgz#62262d6729739f9275723824302fb227c8c93ffb"
+ integrity sha512-K+Iayyo2LtyYhDSYwz5D5QdWw0hCacNzyq1Y821Xna2xSJj7cijoLLYmLxTQgcgZ9mC61nryMy9S7GRbYpI5Ig==
dependencies:
"@babel/code-frame" "^7.0.0"
- ajv "^6.9.1"
+ ajv "^6.10.0"
chalk "^2.1.0"
cross-spawn "^6.0.5"
debug "^4.0.1"
doctrine "^3.0.0"
- eslint-scope "^4.0.3"
- eslint-utils "^1.3.1"
- eslint-visitor-keys "^1.0.0"
- espree "^5.0.1"
+ eslint-scope "^5.0.0"
+ eslint-utils "^1.4.3"
+ eslint-visitor-keys "^1.1.0"
+ espree "^6.1.2"
esquery "^1.0.1"
esutils "^2.0.2"
file-entry-cache "^5.0.1"
functional-red-black-tree "^1.0.1"
- glob "^7.1.2"
- globals "^11.7.0"
+ glob-parent "^5.0.0"
+ globals "^12.1.0"
ignore "^4.0.6"
import-fresh "^3.0.0"
imurmurhash "^0.1.4"
- inquirer "^6.2.2"
- js-yaml "^3.13.0"
+ inquirer "^7.0.0"
+ is-glob "^4.0.0"
+ js-yaml "^3.13.1"
json-stable-stringify-without-jsonify "^1.0.1"
levn "^0.3.0"
- lodash "^4.17.11"
+ lodash "^4.17.14"
minimatch "^3.0.4"
mkdirp "^0.5.1"
natural-compare "^1.4.0"
- optionator "^0.8.2"
- path-is-inside "^1.0.2"
+ optionator "^0.8.3"
progress "^2.0.0"
regexpp "^2.0.1"
- semver "^5.5.1"
- strip-ansi "^4.0.0"
- strip-json-comments "^2.0.1"
+ semver "^6.1.2"
+ strip-ansi "^5.2.0"
+ strip-json-comments "^3.0.1"
table "^5.2.3"
text-table "^0.2.0"
+ v8-compile-cache "^2.0.3"
-espree@^5.0.1:
- version "5.0.1"
- resolved "https://registry.yarnpkg.com/espree/-/espree-5.0.1.tgz#5d6526fa4fc7f0788a5cf75b15f30323e2f81f7a"
- integrity sha512-qWAZcWh4XE/RwzLJejfcofscgMc9CamR6Tn1+XRXNzrvUSSbiAjGOI/fggztjIi7y9VLPqnICMIPiGyr8JaZ0A==
+espree@^6.1.1, espree@^6.1.2:
+ version "6.2.1"
+ resolved "https://registry.yarnpkg.com/espree/-/espree-6.2.1.tgz#77fc72e1fd744a2052c20f38a5b575832e82734a"
+ integrity sha512-ysCxRQY3WaXJz9tdbWOwuWr5Y/XrPTGX9Kiz3yoUXwW0VZ4w30HTkQLaGx/+ttFjF8i+ACbArnB4ce68a9m5hw==
dependencies:
- acorn "^6.0.7"
- acorn-jsx "^5.0.0"
- eslint-visitor-keys "^1.0.0"
+ acorn "^7.1.1"
+ acorn-jsx "^5.2.0"
+ eslint-visitor-keys "^1.1.0"
esprima@^3.1.3:
version "3.1.3"
@@ -4133,6 +5017,11 @@
resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71"
integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==
+espurify@^2.0.1:
+ version "2.0.1"
+ resolved "https://registry.yarnpkg.com/espurify/-/espurify-2.0.1.tgz#c25b3bb613863daa142edcca052370a1a459f41d"
+ integrity sha512-7w/dUrReI/QbJFHRwfomTlkQOXaB1NuCrBRn5Y26HXn5gvh18/19AgLbayVrNxXQfkckvgrJloWyvZDuJ7dhEA==
+
esquery@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/esquery/-/esquery-1.0.1.tgz#406c51658b1f5991a5f9b62b1dc25b00e3e5c708"
@@ -4261,7 +5150,7 @@
debug "*"
path-to-regexp "^1.0.3"
-express@^4.16.2, express@^4.17.1:
+express@^4.17.1:
version "4.17.1"
resolved "https://registry.yarnpkg.com/express/-/express-4.17.1.tgz#4491fc38605cf51f8629d39c2b5d026f98a4c134"
integrity sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==
@@ -4297,6 +5186,13 @@
utils-merge "1.0.1"
vary "~1.1.2"
+ext@^1.1.2:
+ version "1.4.0"
+ resolved "https://registry.yarnpkg.com/ext/-/ext-1.4.0.tgz#89ae7a07158f79d35517882904324077e4379244"
+ integrity sha512-Key5NIsUxdqKg3vIsdw9dSuXpPCQ297y6wBjL30edxwPgt2E44WcWBZey/ZvUc6sERLTxKdyCu4gZFmUbk1Q7A==
+ dependencies:
+ type "^2.0.0"
+
extend-shallow@^2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/extend-shallow/-/extend-shallow-2.0.1.tgz#51af7d614ad9a9f610ea1bafbb989d6b1c56890f"
@@ -4355,7 +5251,17 @@
resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz#7b05218ddf9667bf7f370bf7fdb2cb15fdd0aa49"
integrity sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=
-fast-glob@^2.0.2:
+fast-deep-equal@^3.1.1:
+ version "3.1.1"
+ resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.1.tgz#545145077c501491e33b15ec408c294376e94ae4"
+ integrity sha512-8UEa58QDLauDNfpbrX55Q9jrGHThw2ZMdOky5Gl1CDtVeJDPVrG4Jxx1N8jw2gkWaff5UUuX1KJd+9zGe2B+ZA==
+
+fast-diff@^1.1.2:
+ version "1.2.0"
+ resolved "https://registry.yarnpkg.com/fast-diff/-/fast-diff-1.2.0.tgz#73ee11982d86caaf7959828d519cfe927fac5f03"
+ integrity sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w==
+
+fast-glob@^2.0.2, fast-glob@^2.2.6:
version "2.2.7"
resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-2.2.7.tgz#6953857c3afa475fff92ee6015d52da70a4cd39d"
integrity sha512-g1KuQwHOZAmOZMuBtHdxDtju+T2RT8jgCC9aANsbpdiDDTSnjgfuVsIBNKbUeJI3oKMRExcfNDtJl4OhbffMsw==
@@ -4416,10 +5322,10 @@
resolved "https://registry.yarnpkg.com/figgy-pudding/-/figgy-pudding-3.5.1.tgz#862470112901c727a0e495a80744bd5baa1d6790"
integrity sha512-vNKxJHTEKNThjfrdJwHc7brvM6eVevuO5nTj6ez8ZQ1qbXTvGthucRF7S4vf2cr71QVnT70V34v0S1DyQsti0w==
-figures@^2.0.0:
- version "2.0.0"
- resolved "https://registry.yarnpkg.com/figures/-/figures-2.0.0.tgz#3ab1a2d2a62c8bfb431a0c94cb797a2fce27c962"
- integrity sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI=
+figures@^3.0.0:
+ version "3.2.0"
+ resolved "https://registry.yarnpkg.com/figures/-/figures-3.2.0.tgz#625c18bd293c604dc4a8ddb2febf0c88341746af"
+ integrity sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==
dependencies:
escape-string-regexp "^1.0.5"
@@ -4430,25 +5336,20 @@
dependencies:
flat-cache "^2.0.1"
-file-loader@3.0.1:
- version "3.0.1"
- resolved "https://registry.yarnpkg.com/file-loader/-/file-loader-3.0.1.tgz#f8e0ba0b599918b51adfe45d66d1e771ad560faa"
- integrity sha512-4sNIOXgtH/9WZq4NvlfU3Opn5ynUsqBwSLyM+I7UOwdGigTBYfVVQEwe/msZNX/j4pCJTIM14Fsw66Svo1oVrw==
+file-loader@4.3.0:
+ version "4.3.0"
+ resolved "https://registry.yarnpkg.com/file-loader/-/file-loader-4.3.0.tgz#780f040f729b3d18019f20605f723e844b8a58af"
+ integrity sha512-aKrYPYjF1yG3oX0kWRrqrSMfgftm7oJW5M+m4owoldH5C51C0RkIwB++JbRvEW3IU6/ZG5n8UvEcdgwOt2UOWA==
dependencies:
- loader-utils "^1.0.2"
- schema-utils "^1.0.0"
+ loader-utils "^1.2.3"
+ schema-utils "^2.5.0"
file-uri-to-path@1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd"
integrity sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==
-filesize@3.6.1:
- version "3.6.1"
- resolved "https://registry.yarnpkg.com/filesize/-/filesize-3.6.1.tgz#090bb3ee01b6f801a8a8be99d31710b3422bb317"
- integrity sha512-7KjR1vv6qnicaPMi1iiTcI85CyYwRO/PSFCu6SvqL8jN2Wjt/NIYQTFtFs7fSDCYOstUkEWIQGFUg5YZQfjlcg==
-
-filesize@^6.0.1:
+filesize@6.0.1, filesize@^6.0.1:
version "6.0.1"
resolved "https://registry.yarnpkg.com/filesize/-/filesize-6.0.1.tgz#f850b509909c7c86f7e450ea19006c31c2ed3d2f"
integrity sha512-u4AYWPgbI5GBhs6id1KdImZWn5yfyFrrQ8OWZdN7ZMfA8Bf4HcO0BGo9bmUIEV8yrp8I1xVfJ/dn90GtFNNJcg==
@@ -4463,6 +5364,13 @@
repeat-string "^1.6.1"
to-regex-range "^2.1.0"
+fill-range@^7.0.1:
+ version "7.0.1"
+ resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40"
+ integrity sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==
+ dependencies:
+ to-regex-range "^5.0.1"
+
finalhandler@~1.1.2:
version "1.1.2"
resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.1.2.tgz#b7e7d000ffd11938d0fdb053506f6ebabe9f587d"
@@ -4485,7 +5393,7 @@
mkdirp "^0.5.1"
pkg-dir "^1.0.0"
-find-cache-dir@^2.0.0, find-cache-dir@^2.1.0:
+find-cache-dir@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/find-cache-dir/-/find-cache-dir-2.1.0.tgz#8d0f94cd13fe43c6c7c261a0d86115ca918c05f7"
integrity sha512-Tq6PixE0w/VMFfCgbONnkiQIVol/JJL7nRMi20fqzA4NRs9AfeqMGeRdPi3wIhYkxjeBaWh2rxwapn5Tu3IqOQ==
@@ -4494,12 +5402,27 @@
make-dir "^2.0.0"
pkg-dir "^3.0.0"
-find-up@3.0.0, find-up@^3.0.0:
- version "3.0.0"
- resolved "https://registry.yarnpkg.com/find-up/-/find-up-3.0.0.tgz#49169f1d7993430646da61ecc5ae355c21c97b73"
- integrity sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==
+find-cache-dir@^3.2.0, find-cache-dir@^3.3.1:
+ version "3.3.1"
+ resolved "https://registry.yarnpkg.com/find-cache-dir/-/find-cache-dir-3.3.1.tgz#89b33fad4a4670daa94f855f7fbe31d6d84fe880"
+ integrity sha512-t2GDMt3oGC/v+BMwzmllWDuJF/xcDtE5j/fCGbqDD7OLuJkj0cfh1YSA5VKPvwMeLFLNDBkwOKZ2X85jGLVftQ==
dependencies:
- locate-path "^3.0.0"
+ commondir "^1.0.1"
+ make-dir "^3.0.2"
+ pkg-dir "^4.1.0"
+
+find-root@^1.1.0:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/find-root/-/find-root-1.1.0.tgz#abcfc8ba76f708c42a97b3d685b7e9450bfb9ce4"
+ integrity sha512-NKfW6bec6GfKc0SGx1e07QZY9PE99u0Bft/0rzSD5k3sO/vwkVUpDUKVm5Gpp5Ue3YfShPFTX2070tDs5kB9Ng==
+
+find-up@4.1.0, find-up@^4.0.0, find-up@^4.1.0:
+ version "4.1.0"
+ resolved "https://registry.yarnpkg.com/find-up/-/find-up-4.1.0.tgz#97afe7d6cdc0bc5928584b7c8d7b16e8a9aa5d19"
+ integrity sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==
+ dependencies:
+ locate-path "^5.0.0"
+ path-exists "^4.0.0"
find-up@^1.0.0:
version "1.1.2"
@@ -4516,6 +5439,13 @@
dependencies:
locate-path "^2.0.0"
+find-up@^3.0.0:
+ version "3.0.0"
+ resolved "https://registry.yarnpkg.com/find-up/-/find-up-3.0.0.tgz#49169f1d7993430646da61ecc5ae355c21c97b73"
+ integrity sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==
+ dependencies:
+ locate-path "^3.0.0"
+
flat-cache@^2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-2.0.1.tgz#5d296d6f04bda44a4630a301413bdbc2ec085ec0"
@@ -4574,26 +5504,19 @@
dependencies:
for-in "^1.0.1"
-for-own@^1.0.0:
- version "1.0.0"
- resolved "https://registry.yarnpkg.com/for-own/-/for-own-1.0.0.tgz#c63332f415cedc4b04dbfe70cf836494c53cb44b"
- integrity sha1-xjMy9BXO3EsE2/5wz4NklMU8tEs=
- dependencies:
- for-in "^1.0.1"
-
forever-agent@~0.6.1:
version "0.6.1"
resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91"
integrity sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=
-fork-ts-checker-webpack-plugin@1.5.0:
- version "1.5.0"
- resolved "https://registry.yarnpkg.com/fork-ts-checker-webpack-plugin/-/fork-ts-checker-webpack-plugin-1.5.0.tgz#ce1d77190b44d81a761b10b6284a373795e41f0c"
- integrity sha512-zEhg7Hz+KhZlBhILYpXy+Beu96gwvkROWJiTXOCyOOMMrdBIRPvsBpBqgTI4jfJGrJXcqGwJR8zsBGDmzY0jsA==
+fork-ts-checker-webpack-plugin@3.1.1:
+ version "3.1.1"
+ resolved "https://registry.yarnpkg.com/fork-ts-checker-webpack-plugin/-/fork-ts-checker-webpack-plugin-3.1.1.tgz#a1642c0d3e65f50c2cc1742e9c0a80f441f86b19"
+ integrity sha512-DuVkPNrM12jR41KM2e+N+styka0EgLkTnXmNcXdgOM37vtGeY+oCBK/Jx0hzSeEU6memFCtWb4htrHPMDfwwUQ==
dependencies:
babel-code-frame "^6.22.0"
chalk "^2.4.1"
- chokidar "^2.0.4"
+ chokidar "^3.3.0"
micromatch "^3.1.10"
minimatch "^3.0.4"
semver "^5.6.0"
@@ -4634,15 +5557,6 @@
inherits "^2.0.1"
readable-stream "^2.0.0"
-fs-extra@7.0.1, fs-extra@^7.0.0:
- version "7.0.1"
- resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-7.0.1.tgz#4f189c44aa123b895f722804f55ea23eadc348e9"
- integrity sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==
- dependencies:
- graceful-fs "^4.1.2"
- jsonfile "^4.0.0"
- universalify "^0.1.0"
-
fs-extra@^4.0.2:
version "4.0.3"
resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-4.0.3.tgz#0d852122e5bc5beb453fb028e9c0c9bf36340c94"
@@ -4652,6 +5566,41 @@
jsonfile "^4.0.0"
universalify "^0.1.0"
+fs-extra@^7.0.0:
+ version "7.0.1"
+ resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-7.0.1.tgz#4f189c44aa123b895f722804f55ea23eadc348e9"
+ integrity sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==
+ dependencies:
+ graceful-fs "^4.1.2"
+ jsonfile "^4.0.0"
+ universalify "^0.1.0"
+
+fs-extra@^8.1.0:
+ version "8.1.0"
+ resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-8.1.0.tgz#49d43c45a88cd9677668cb7be1b46efdb8d2e1c0"
+ integrity sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==
+ dependencies:
+ graceful-fs "^4.2.0"
+ jsonfile "^4.0.0"
+ universalify "^0.1.0"
+
+fs-extra@^9.0.0:
+ version "9.0.0"
+ resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.0.0.tgz#b6afc31036e247b2466dc99c29ae797d5d4580a3"
+ integrity sha512-pmEYSk3vYsG/bF651KPUXZ+hvjpgWYw/Gc7W9NFUe3ZVLczKKWIij3IKpOrQcdw4TILtibFslZ0UmR8Vvzig4g==
+ dependencies:
+ at-least-node "^1.0.0"
+ graceful-fs "^4.2.0"
+ jsonfile "^6.0.1"
+ universalify "^1.0.0"
+
+fs-minipass@^2.0.0:
+ version "2.1.0"
+ resolved "https://registry.yarnpkg.com/fs-minipass/-/fs-minipass-2.1.0.tgz#7f5036fdbf12c63c169190cbe4199c852271f9fb"
+ integrity sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==
+ dependencies:
+ minipass "^3.0.0"
+
fs-write-stream-atomic@^1.0.8:
version "1.0.10"
resolved "https://registry.yarnpkg.com/fs-write-stream-atomic/-/fs-write-stream-atomic-1.0.10.tgz#b47df53493ef911df75731e70a9ded0189db40c9"
@@ -4667,10 +5616,10 @@
resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8=
-fsevents@2.0.6:
- version "2.0.6"
- resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.0.6.tgz#87b19df0bfb4a1a51d7ddb51b01b5f3bedb40c33"
- integrity sha512-vfmKZp3XPM36DNF0qhW+Cdxk7xm7gTEHY1clv1Xq1arwRQuKZgAhw+NZNWbJBtuaNxzNXwhfdPYRrvIbjfS33A==
+fsevents@2.1.2:
+ version "2.1.2"
+ resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.1.2.tgz#4c0a1fb34bc68e543b4b82a9ec392bfbda840805"
+ integrity sha512-R4wDiBwZ0KzpgOWetKDug1FZcYhqYnUYKtfZYt4mD5SBz76q0KR4Q9o7GIPamsVPGmW3EYPPJ0dOOjvx32ldZA==
fsevents@^1.2.7:
version "1.2.11"
@@ -4680,6 +5629,11 @@
bindings "^1.5.0"
nan "^2.12.1"
+fsevents@~2.1.2:
+ version "2.1.3"
+ resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.1.3.tgz#fb738703ae8d2f9fe900c33836ddebee8b97f23e"
+ integrity sha512-Auw9a4AxqWpa9GUfj370BMPzzyncfBABW8Mab7BGWBYDj4Isgq+cDKtx0i6u9jcX9pQDnswsaaOTgTmA5pEjuQ==
+
function-bind@^1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d"
@@ -4690,6 +5644,11 @@
resolved "https://registry.yarnpkg.com/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz#1b0ab3bd553b2a0d6399d29c0e3ea0b252078327"
integrity sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=
+gensync@^1.0.0-beta.1:
+ version "1.0.0-beta.1"
+ resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.1.tgz#58f4361ff987e5ff6e1e7a210827aa371eaac269"
+ integrity sha512-r8EC6NO1sngH/zdD9fiRDLdcgnbayXah+mLgManTaIZJqEC1MZstmnox8KpnI2/fxQwrp5OpCOYWLp4rBl4Jcg==
+
get-caller-file@^1.0.1:
version "1.0.3"
resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.3.tgz#f978fa4c90d1dfe7ff2d6beda2a515e713bdcf4a"
@@ -4705,6 +5664,21 @@
resolved "https://registry.yarnpkg.com/get-own-enumerable-property-symbols/-/get-own-enumerable-property-symbols-3.0.2.tgz#b5fde77f22cbe35f390b4e089922c50bce6ef664"
integrity sha512-I0UBV/XOz1XkIJHEUDMZAbzCThU/H8DxmSfmdGcKPnVhu2VfFqr34jr9777IyaTYvxjedWhqVIilEDsCdP5G6g==
+get-set-props@^0.1.0:
+ version "0.1.0"
+ resolved "https://registry.yarnpkg.com/get-set-props/-/get-set-props-0.1.0.tgz#998475c178445686d0b32246da5df8dbcfbe8ea3"
+ integrity sha1-mYR1wXhEVobQsyJG2l3428++jqM=
+
+get-stdin@^6.0.0:
+ version "6.0.0"
+ resolved "https://registry.yarnpkg.com/get-stdin/-/get-stdin-6.0.0.tgz#9e09bf712b360ab9225e812048f71fde9c89657b"
+ integrity sha512-jp4tHawyV7+fkkSKyvjuLZswblUtz+SQKzSWnBbii16BuZksJlU1wuBYXY75r+duh/llF1ur6oNwi+2ZzjKZ7g==
+
+get-stdin@^7.0.0:
+ version "7.0.0"
+ resolved "https://registry.yarnpkg.com/get-stdin/-/get-stdin-7.0.0.tgz#8d5de98f15171a125c5e516643c7a6d0ea8a96f6"
+ integrity sha512-zRKcywvrXlXsA0v0i9Io4KDRaAw7+a1ZpjRwl9Wox8PFlVCCHra7E9c4kqXCoCM9nR5tBkaTTZRBoCm60bFqTQ==
+
get-stream@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-3.0.0.tgz#8e943d1358dc37555054ecbe2edb05aa174ede14"
@@ -4744,12 +5718,19 @@
is-glob "^3.1.0"
path-dirname "^1.0.0"
+glob-parent@^5.0.0, glob-parent@~5.1.0:
+ version "5.1.1"
+ resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.1.tgz#b6c1ef417c4e5663ea498f1c45afac6916bbc229"
+ integrity sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==
+ dependencies:
+ is-glob "^4.0.1"
+
glob-to-regexp@^0.3.0:
version "0.3.0"
resolved "https://registry.yarnpkg.com/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz#8c5a1494d2066c570cc3bfe4496175acc4d502ab"
integrity sha1-jFoUlNIGbFcMw7/kSWF1rMTVAqs=
-glob@^7.0.3, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4:
+glob@^7.0.3, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6:
version "7.1.6"
resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.6.tgz#141f33b81a7c2492e125594307480c46679278a6"
integrity sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==
@@ -4768,6 +5749,13 @@
dependencies:
ini "^1.3.4"
+global-dirs@^2.0.1:
+ version "2.0.1"
+ resolved "https://registry.yarnpkg.com/global-dirs/-/global-dirs-2.0.1.tgz#acdf3bb6685bcd55cb35e8a052266569e9469201"
+ integrity sha512-5HqUqdhkEovj2Of/ms3IeS/EekcO54ytHRLV4PEY2rhRwrHXLQjeVEES0Lhka0xwNDtGYn58wyC4s5+MHsOO6A==
+ dependencies:
+ ini "^1.3.5"
+
global-modules@2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/global-modules/-/global-modules-2.0.0.tgz#997605ad2345f27f51539bea26574421215c7780"
@@ -4784,11 +5772,18 @@
kind-of "^6.0.2"
which "^1.3.1"
-globals@^11.1.0, globals@^11.7.0:
+globals@^11.1.0:
version "11.12.0"
resolved "https://registry.yarnpkg.com/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e"
integrity sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==
+globals@^12.1.0:
+ version "12.4.0"
+ resolved "https://registry.yarnpkg.com/globals/-/globals-12.4.0.tgz#a18813576a41b00a24a97e7f815918c2e19925f8"
+ integrity sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==
+ dependencies:
+ type-fest "^0.8.1"
+
globby@8.0.2:
version "8.0.2"
resolved "https://registry.yarnpkg.com/globby/-/globby-8.0.2.tgz#5697619ccd95c5275dbb2d6faa42087c1a941d8d"
@@ -4813,6 +5808,20 @@
pify "^2.0.0"
pinkie-promise "^2.0.0"
+globby@^9.0.0:
+ version "9.2.0"
+ resolved "https://registry.yarnpkg.com/globby/-/globby-9.2.0.tgz#fd029a706c703d29bdd170f4b6db3a3f7a7cb63d"
+ integrity sha512-ollPHROa5mcxDEkwg6bPt3QbEf4pDQSNtd6JPL1YvOvAo/7/0VAm9TccUeoTmarjPw4pfUthSCqcyfNB1I3ZSg==
+ dependencies:
+ "@types/glob" "^7.1.1"
+ array-union "^1.0.2"
+ dir-glob "^2.2.2"
+ fast-glob "^2.2.6"
+ glob "^7.1.3"
+ ignore "^4.0.3"
+ pify "^4.0.1"
+ slash "^2.0.0"
+
got@^9.6.0:
version "9.6.0"
resolved "https://registry.yarnpkg.com/got/-/got-9.6.0.tgz#edf45e7d67f99545705de1f7bbeeeb121765ed85"
@@ -4835,6 +5844,11 @@
resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.3.tgz#4a12ff1b60376ef09862c2093edd908328be8423"
integrity sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==
+graceful-fs@^4.2.0, graceful-fs@^4.2.2:
+ version "4.2.4"
+ resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.4.tgz#2256bde14d3632958c465ebc96dc467ca07a29fb"
+ integrity sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw==
+
growly@^1.3.0:
version "1.3.0"
resolved "https://registry.yarnpkg.com/growly/-/growly-1.3.0.tgz#f10748cbe76af964b7c96c93c6bcc28af120c081"
@@ -4904,6 +5918,11 @@
resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd"
integrity sha1-tdRU3CGZriJWmfNGfloH87lVuv0=
+has-flag@^4.0.0:
+ version "4.0.0"
+ resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b"
+ integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==
+
has-symbols@^1.0.0, has-symbols@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.1.tgz#9f5214758a44196c406d9bd76cebf81ec2dd31e8"
@@ -4968,7 +5987,7 @@
inherits "^2.0.3"
minimalistic-assert "^1.0.1"
-he@1.2.x:
+he@^1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f"
integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==
@@ -5048,34 +6067,34 @@
dependencies:
whatwg-encoding "^1.0.1"
-html-entities@^1.2.0:
- version "1.2.1"
- resolved "https://registry.yarnpkg.com/html-entities/-/html-entities-1.2.1.tgz#0df29351f0721163515dfb9e5543e5f6eed5162f"
- integrity sha1-DfKTUfByEWNRXfueVUPl9u7VFi8=
+html-entities@^1.2.1:
+ version "1.3.1"
+ resolved "https://registry.yarnpkg.com/html-entities/-/html-entities-1.3.1.tgz#fb9a1a4b5b14c5daba82d3e34c6ae4fe701a0e44"
+ integrity sha512-rhE/4Z3hIhzHAUKbW8jVcCyuT5oJCXXqhN/6mXXVCpzTmvJnoH2HL/bt3EZ6p55jbFJBeAe1ZNpL5BugLujxNA==
-html-minifier@^3.5.20:
- version "3.5.21"
- resolved "https://registry.yarnpkg.com/html-minifier/-/html-minifier-3.5.21.tgz#d0040e054730e354db008463593194015212d20c"
- integrity sha512-LKUKwuJDhxNa3uf/LPR/KVjm/l3rBqtYeCOAekvG8F1vItxMUpueGd94i/asDDr8/1u7InxzFA5EeGjhhG5mMA==
+html-minifier-terser@^5.0.1:
+ version "5.1.0"
+ resolved "https://registry.yarnpkg.com/html-minifier-terser/-/html-minifier-terser-5.1.0.tgz#95d3df037f04835e9d1a09d1767c0e361a7de916"
+ integrity sha512-tiYE76O1zunboByeB/nFGwUEb263Z3nkNv6Lz2oLC1s6M36bLKfTrjQ+7ssVfaucVllE+N7hh/FbpbxvnIA+LQ==
dependencies:
- camel-case "3.0.x"
- clean-css "4.2.x"
- commander "2.17.x"
- he "1.2.x"
- param-case "2.1.x"
- relateurl "0.2.x"
- uglify-js "3.4.x"
+ camel-case "^4.1.1"
+ clean-css "^4.2.3"
+ commander "^4.1.1"
+ he "^1.2.0"
+ param-case "^3.0.3"
+ relateurl "^0.2.7"
+ terser "^4.6.3"
-html-webpack-plugin@4.0.0-beta.5:
- version "4.0.0-beta.5"
- resolved "https://registry.yarnpkg.com/html-webpack-plugin/-/html-webpack-plugin-4.0.0-beta.5.tgz#2c53083c1151bfec20479b1f8aaf0039e77b5513"
- integrity sha512-y5l4lGxOW3pz3xBTFdfB9rnnrWRPVxlAhX6nrBYIcW+2k2zC3mSp/3DxlWVCMBfnO6UAnoF8OcFn0IMy6kaKAQ==
+html-webpack-plugin@4.0.0-beta.11:
+ version "4.0.0-beta.11"
+ resolved "https://registry.yarnpkg.com/html-webpack-plugin/-/html-webpack-plugin-4.0.0-beta.11.tgz#3059a69144b5aecef97708196ca32f9e68677715"
+ integrity sha512-4Xzepf0qWxf8CGg7/WQM5qBB2Lc/NFI7MhU59eUDTkuQp3skZczH4UA1d6oQyDEIoMDgERVhRyTdtUPZ5s5HBg==
dependencies:
- html-minifier "^3.5.20"
- loader-utils "^1.1.0"
- lodash "^4.17.11"
+ html-minifier-terser "^5.0.1"
+ loader-utils "^1.2.3"
+ lodash "^4.17.15"
pretty-error "^2.1.1"
- tapable "^1.1.0"
+ tapable "^1.1.3"
util.promisify "1.0.0"
htmlparser2@^3.3.0:
@@ -5137,7 +6156,7 @@
resolved "https://registry.yarnpkg.com/http-parser-js/-/http-parser-js-0.4.10.tgz#92c9c1374c35085f75db359ec56cc257cbb93fa4"
integrity sha1-ksnBN0w1CF912zWexWzCV8u5P6Q=
-http-proxy-middleware@^0.19.1:
+http-proxy-middleware@0.19.1:
version "0.19.1"
resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-0.19.1.tgz#183c7dc4aa1479150306498c210cdaf96080a43a"
integrity sha512-yHYTgWMQO8VvwNS22eLLloAkvungsKdKTLO8AJlftYIKNfJr3GK3zK0ZCfzDDGUBttdGc8xFy1mCitvNKQtC3Q==
@@ -5177,12 +6196,7 @@
dependencies:
safer-buffer ">= 2.1.2 < 3"
-icss-replace-symbols@^1.1.0:
- version "1.1.0"
- resolved "https://registry.yarnpkg.com/icss-replace-symbols/-/icss-replace-symbols-1.1.0.tgz#06ea6f83679a7749e386cfe1fe812ae5db223ded"
- integrity sha1-Bupvg2ead0njhs/h/oEq5dsiPe0=
-
-icss-utils@^4.1.0:
+icss-utils@^4.0.0, icss-utils@^4.1.1:
version "4.1.1"
resolved "https://registry.yarnpkg.com/icss-utils/-/icss-utils-4.1.1.tgz#21170b53789ee27447c2f47dd683081403f9a467"
integrity sha512-4aFq7wvWyMHKgxsH8QQtGpvbASCf+eM3wPRLI6R+MgAnTCZ6STYsRvttLvRWK0Nfif5piF394St3HeJDaljGPA==
@@ -5211,11 +6225,16 @@
resolved "https://registry.yarnpkg.com/ignore/-/ignore-3.3.10.tgz#0a97fb876986e8081c631160f8f9f389157f0043"
integrity sha512-Pgs951kaMm5GXP7MOvxERINe3gsaVjUWFm+UZPSq9xYriQAksyhg0csnS0KXSNRD5NmNdapXEpjxG49+AKh/ug==
-ignore@^4.0.6:
+ignore@^4.0.3, ignore@^4.0.6:
version "4.0.6"
resolved "https://registry.yarnpkg.com/ignore/-/ignore-4.0.6.tgz#750e3db5862087b4737ebac8207ffd1ef27b25fc"
integrity sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==
+ignore@^5.0.5, ignore@^5.1.1:
+ version "5.1.4"
+ resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.1.4.tgz#84b7b3dbe64552b6ef0eca99f6743dbec6d97adf"
+ integrity sha512-MzbUSahkTW1u7JpKKjY7LCARd1fU5W2rLdxlM4kdkayuCwZImjkpluF9CM1aLewYJguPDqewLam18Y6AU69A8A==
+
image-size@~0.5.0:
version "0.5.5"
resolved "https://registry.yarnpkg.com/image-size/-/image-size-0.5.5.tgz#09dfd4ab9d20e29eb1c3e80b8990378df9e3cb9c"
@@ -5251,7 +6270,7 @@
caller-path "^2.0.0"
resolve-from "^3.0.0"
-import-fresh@^3.0.0:
+import-fresh@^3.0.0, import-fresh@^3.1.0:
version "3.2.1"
resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.2.1.tgz#633ff618506e793af5ac91bf48b72677e15cbe66"
integrity sha512-6e1q1cnWP2RXD9/keSkxHScg508CdXqXWgWBaETNhyuBFz+kUZlKboh+ISK+bU++DmbHimVBrOz/zzPe0sZ3sQ==
@@ -5279,17 +6298,32 @@
pkg-dir "^3.0.0"
resolve-cwd "^2.0.0"
+import-modules@^2.0.0:
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/import-modules/-/import-modules-2.0.0.tgz#9c1e13b4e7a15682f70a6e3fa29534e4540cfc5d"
+ integrity sha512-iczM/v9drffdNnABOKwj0f9G3cFDon99VcG1mxeBsdqnbd+vnQ5c2uAiCHNQITqFTOPaEvwg3VjoWCur0uHLEw==
+
imurmurhash@^0.1.4:
version "0.1.4"
resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea"
integrity sha1-khi5srkoojixPcT7a21XbyMUU+o=
+indent-string@^3.0.0:
+ version "3.2.0"
+ resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-3.2.0.tgz#4a5fd6d27cc332f37e5419a504dbb837105c9289"
+ integrity sha1-Sl/W0nzDMvN+VBmlBNu4NxBckok=
+
+indent-string@^4.0.0:
+ version "4.0.0"
+ resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-4.0.0.tgz#624f8f4497d619b2d9768531d58f4122854d7251"
+ integrity sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==
+
indexes-of@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/indexes-of/-/indexes-of-1.0.1.tgz#f30f716c8e2bd346c7b67d3df3915566a7c05607"
integrity sha1-8w9xbI4r00bHtn0985FVZqfAVgc=
-infer-owner@^1.0.3:
+infer-owner@^1.0.3, infer-owner@^1.0.4:
version "1.0.4"
resolved "https://registry.yarnpkg.com/infer-owner/-/infer-owner-1.0.4.tgz#c4cefcaa8e51051c2a40ba2ce8a3d27295af9467"
integrity sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A==
@@ -5322,45 +6356,45 @@
resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.5.tgz#eee25f56db1c9ec6085e0c22778083f596abf927"
integrity sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==
-inquirer@6.5.0:
- version "6.5.0"
- resolved "https://registry.yarnpkg.com/inquirer/-/inquirer-6.5.0.tgz#2303317efc9a4ea7ec2e2df6f86569b734accf42"
- integrity sha512-scfHejeG/lVZSpvCXpsB4j/wQNPM5JC8kiElOI0OUTwmc1RTpXr4H32/HOlQHcZiYl2z2VElwuCVDRG8vFmbnA==
+inquirer@7.0.4:
+ version "7.0.4"
+ resolved "https://registry.yarnpkg.com/inquirer/-/inquirer-7.0.4.tgz#99af5bde47153abca23f5c7fc30db247f39da703"
+ integrity sha512-Bu5Td5+j11sCkqfqmUTiwv+tWisMtP0L7Q8WrqA2C/BbBhy1YTdFrvjjlrKq8oagA/tLQBski2Gcx/Sqyi2qSQ==
dependencies:
- ansi-escapes "^3.2.0"
+ ansi-escapes "^4.2.1"
chalk "^2.4.2"
- cli-cursor "^2.1.0"
+ cli-cursor "^3.1.0"
cli-width "^2.0.0"
external-editor "^3.0.3"
- figures "^2.0.0"
- lodash "^4.17.12"
- mute-stream "0.0.7"
+ figures "^3.0.0"
+ lodash "^4.17.15"
+ mute-stream "0.0.8"
run-async "^2.2.0"
- rxjs "^6.4.0"
- string-width "^2.1.0"
+ rxjs "^6.5.3"
+ string-width "^4.1.0"
strip-ansi "^5.1.0"
through "^2.3.6"
-inquirer@^6.2.2:
- version "6.5.2"
- resolved "https://registry.yarnpkg.com/inquirer/-/inquirer-6.5.2.tgz#ad50942375d036d327ff528c08bd5fab089928ca"
- integrity sha512-cntlB5ghuB0iuO65Ovoi8ogLHiWGs/5yNrtUcKjFhSSiVeAIVpD7koaSU9RM8mpXw5YDi9RdYXGQMaOURB7ycQ==
+inquirer@^7.0.0:
+ version "7.1.0"
+ resolved "https://registry.yarnpkg.com/inquirer/-/inquirer-7.1.0.tgz#1298a01859883e17c7264b82870ae1034f92dd29"
+ integrity sha512-5fJMWEmikSYu0nv/flMc475MhGbB7TSPd/2IpFV4I4rMklboCH2rQjYY5kKiYGHqUF9gvaambupcJFFG9dvReg==
dependencies:
- ansi-escapes "^3.2.0"
- chalk "^2.4.2"
- cli-cursor "^2.1.0"
+ ansi-escapes "^4.2.1"
+ chalk "^3.0.0"
+ cli-cursor "^3.1.0"
cli-width "^2.0.0"
external-editor "^3.0.3"
- figures "^2.0.0"
- lodash "^4.17.12"
- mute-stream "0.0.7"
- run-async "^2.2.0"
- rxjs "^6.4.0"
- string-width "^2.1.0"
- strip-ansi "^5.1.0"
+ figures "^3.0.0"
+ lodash "^4.17.15"
+ mute-stream "0.0.8"
+ run-async "^2.4.0"
+ rxjs "^6.5.3"
+ string-width "^4.1.0"
+ strip-ansi "^6.0.0"
through "^2.3.6"
-internal-ip@^4.2.0:
+internal-ip@^4.3.0:
version "4.3.0"
resolved "https://registry.yarnpkg.com/internal-ip/-/internal-ip-4.3.0.tgz#845452baad9d2ca3b69c635a137acb9a0dad0907"
integrity sha512-S1zBo1D6zcsyuC6PMmY5+55YMILQ9av8lotMx447Bq6SAgo/sDK6y6uUKmuYhW7eacnIhFfsPmCNYdDzsnnDCg==
@@ -5368,6 +6402,20 @@
default-gateway "^4.2.0"
ipaddr.js "^1.9.0"
+internal-slot@^1.0.2:
+ version "1.0.2"
+ resolved "https://registry.yarnpkg.com/internal-slot/-/internal-slot-1.0.2.tgz#9c2e9fb3cd8e5e4256c6f45fe310067fcfa378a3"
+ integrity sha512-2cQNfwhAfJIkU4KZPkDI+Gj5yNNnbqi40W9Gge6dfnk4TocEVm00B3bdiL+JINrbGJil2TeHvM4rETGzk/f/0g==
+ dependencies:
+ es-abstract "^1.17.0-next.1"
+ has "^1.0.3"
+ side-channel "^1.0.2"
+
+interpret@^1.2.0:
+ version "1.2.0"
+ resolved "https://registry.yarnpkg.com/interpret/-/interpret-1.2.0.tgz#d5061a6224be58e8083985f5014d844359576296"
+ integrity sha512-mT34yGKMNceBQUoVn7iCDKDntA7SC6gycMAWzGx1z/CMCTV7b2AAtXlo3nRyHZ1FelRkQbQjprHSYGwzLtkVbw==
+
invariant@^2.2.2, invariant@^2.2.4:
version "2.2.4"
resolved "https://registry.yarnpkg.com/invariant/-/invariant-2.2.4.tgz#610f3c92c9359ce1db616e538008d23ff35158e6"
@@ -5400,11 +6448,29 @@
resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz#bff38543eeb8984825079ff3a2a8e6cbd46781b3"
integrity sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==
+irregular-plurals@^2.0.0:
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/irregular-plurals/-/irregular-plurals-2.0.0.tgz#39d40f05b00f656d0b7fa471230dd3b714af2872"
+ integrity sha512-Y75zBYLkh0lJ9qxeHlMjQ7bSbyiSqNW/UOPWDmzC7cXskL1hekSITh1Oc6JV0XCWWZ9DE8VYSB71xocLk3gmGw==
+
is-absolute-url@^2.0.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/is-absolute-url/-/is-absolute-url-2.1.0.tgz#50530dfb84fcc9aa7dbe7852e83a37b93b9f2aa6"
integrity sha1-UFMN+4T8yap9vnhS6Do3uTufKqY=
+is-absolute-url@^3.0.3:
+ version "3.0.3"
+ resolved "https://registry.yarnpkg.com/is-absolute-url/-/is-absolute-url-3.0.3.tgz#96c6a22b6a23929b11ea0afb1836c36ad4a5d698"
+ integrity sha512-opmNIX7uFnS96NtPmhWQgQx6/NYFgsUXYMllcfzwWKUMwfo8kku1TvE6hkNcH+Q1ts5cMVrsY7j0bxXQDciu9Q==
+
+is-absolute@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/is-absolute/-/is-absolute-1.0.0.tgz#395e1ae84b11f26ad1795e73c17378e48a301576"
+ integrity sha512-dOWoqflvcydARa360Gvv18DZ/gRuHKi2NU/wU5X1ZFzdYfH29nkiNZsF3mp4OJ3H4yo9Mx8A/uAGNzpzPN3yBA==
+ dependencies:
+ is-relative "^1.0.0"
+ is-windows "^1.0.1"
+
is-accessor-descriptor@^0.1.6:
version "0.1.6"
resolved "https://registry.yarnpkg.com/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz#a9e12cb3ae8d876727eeef3843f8a0897b5c98d6"
@@ -5441,6 +6507,13 @@
dependencies:
binary-extensions "^1.0.0"
+is-binary-path@~2.1.0:
+ version "2.1.0"
+ resolved "https://registry.yarnpkg.com/is-binary-path/-/is-binary-path-2.1.0.tgz#ea1f7f3b80f064236e83470f86c09c254fb45b09"
+ integrity sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==
+ dependencies:
+ binary-extensions "^2.0.0"
+
is-buffer@^1.0.2, is-buffer@^1.1.5:
version "1.1.6"
resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be"
@@ -5517,6 +6590,16 @@
resolved "https://registry.yarnpkg.com/is-directory/-/is-directory-0.3.1.tgz#61339b6f2475fc772fd9c9d83f5c8575dc154ae1"
integrity sha1-YTObbyR1/Hcv2cnYP1yFddwVSuE=
+is-docker@^2.0.0:
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/is-docker/-/is-docker-2.0.0.tgz#2cb0df0e75e2d064fe1864c37cdeacb7b2dcf25b"
+ integrity sha512-pJEdRugimx4fBMra5z2/5iRdZ63OhYV0vr0Dwm5+xtW4D1FvRkB8hamMIhnWfyJeDdyr/aa7BDyNbtG38VxgoQ==
+
+is-error@^2.2.0:
+ version "2.2.2"
+ resolved "https://registry.yarnpkg.com/is-error/-/is-error-2.2.2.tgz#c10ade187b3c93510c5470a5567833ee25649843"
+ integrity sha512-IOQqts/aHWbiisY5DuPJQ0gcbvaLFCa7fBa9xoLfxBZvQ+ZI/Zh9xoI7Gk+G64N0FdK4AbibytHht2tWgpJWLg==
+
is-extendable@^0.1.0, is-extendable@^0.1.1:
version "0.1.1"
resolved "https://registry.yarnpkg.com/is-extendable/-/is-extendable-0.1.1.tgz#62b110e289a471418e3ec36a617d472e301dfc89"
@@ -5546,11 +6629,24 @@
resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz#a3b30a5c4f199183167aaab93beefae3ddfb654f"
integrity sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=
+is-fullwidth-code-point@^3.0.0:
+ version "3.0.0"
+ resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d"
+ integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==
+
is-generator-fn@^2.0.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/is-generator-fn/-/is-generator-fn-2.1.0.tgz#7d140adc389aaf3011a8f2a2a4cfa6faadffb118"
integrity sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==
+is-get-set-prop@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/is-get-set-prop/-/is-get-set-prop-1.0.0.tgz#2731877e4d78a6a69edcce6bb9d68b0779e76312"
+ integrity sha1-JzGHfk14pqae3M5rudaLB3nnYxI=
+ dependencies:
+ get-set-props "^0.1.0"
+ lowercase-keys "^1.0.0"
+
is-glob@^3.1.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-3.1.0.tgz#7ba5ae24217804ac70707b96922567486cc3e84a"
@@ -5558,7 +6654,7 @@
dependencies:
is-extglob "^2.1.0"
-is-glob@^4.0.0:
+is-glob@^4.0.0, is-glob@^4.0.1, is-glob@~4.0.1:
version "4.0.1"
resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.1.tgz#7567dbe9f2f5e2467bc77ab83c4a29482407a5dc"
integrity sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==
@@ -5573,16 +6669,41 @@
global-dirs "^0.1.0"
is-path-inside "^1.0.0"
+is-installed-globally@^0.3.1:
+ version "0.3.2"
+ resolved "https://registry.yarnpkg.com/is-installed-globally/-/is-installed-globally-0.3.2.tgz#fd3efa79ee670d1187233182d5b0a1dd00313141"
+ integrity sha512-wZ8x1js7Ia0kecP/CHM/3ABkAmujX7WPvQk6uu3Fly/Mk44pySulQpnHG46OMjHGXApINnV4QhY3SWnECO2z5g==
+ dependencies:
+ global-dirs "^2.0.1"
+ is-path-inside "^3.0.1"
+
+is-js-type@^2.0.0:
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/is-js-type/-/is-js-type-2.0.0.tgz#73617006d659b4eb4729bba747d28782df0f7e22"
+ integrity sha1-c2FwBtZZtOtHKbunR9KHgt8PfiI=
+ dependencies:
+ js-types "^1.0.0"
+
is-mobile@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/is-mobile/-/is-mobile-2.1.0.tgz#4c0cab72f3358dd9986007389b30729fae80da0b"
integrity sha512-M5OhlZwh+aTlmRUvDg0Wq3uWVNa+w4DyZ2SjbrS+BhSLu9Po+JXHendC305ZEu+Hh7lywb19Zu4kYXu3L1Oo8A==
+is-negated-glob@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/is-negated-glob/-/is-negated-glob-1.0.0.tgz#6910bca5da8c95e784b5751b976cf5a10fee36d2"
+ integrity sha1-aRC8pdqMleeEtXUbl2z1oQ/uNtI=
+
is-npm@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/is-npm/-/is-npm-3.0.0.tgz#ec9147bfb629c43f494cf67936a961edec7e8053"
integrity sha512-wsigDr1Kkschp2opC4G3yA6r9EgVA6NjRpWzIi9axXqeIaAATPRJc4uLujXe3Nd9uO8KoDyA4MD6aZSeXTADhA==
+is-npm@^4.0.0:
+ version "4.0.0"
+ resolved "https://registry.yarnpkg.com/is-npm/-/is-npm-4.0.0.tgz#c90dd8380696df87a7a6d823c20d0b12bbe3c84d"
+ integrity sha512-96ECIfh9xtDDlPylNPXhzjsykHsMJZ18ASpaWzQyBr4YRTcVjUvzaHayDAES2oU/3KpljhHUjtSRNiDwi0F0ig==
+
is-number@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/is-number/-/is-number-3.0.0.tgz#24fd6201a4782cf50561c810276afc7d12d71195"
@@ -5590,22 +6711,40 @@
dependencies:
kind-of "^3.0.2"
+is-number@^7.0.0:
+ version "7.0.0"
+ resolved "https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b"
+ integrity sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==
+
+is-obj-prop@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/is-obj-prop/-/is-obj-prop-1.0.0.tgz#b34de79c450b8d7c73ab2cdf67dc875adb85f80e"
+ integrity sha1-s03nnEULjXxzqyzfZ9yHWtuF+A4=
+ dependencies:
+ lowercase-keys "^1.0.0"
+ obj-props "^1.0.0"
+
is-obj@^1.0.0, is-obj@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/is-obj/-/is-obj-1.0.1.tgz#3e4729ac1f5fde025cd7d83a896dab9f4f67db0f"
integrity sha1-PkcprB9f3gJc19g6iW2rn09n2w8=
-is-path-cwd@^1.0.0:
- version "1.0.0"
- resolved "https://registry.yarnpkg.com/is-path-cwd/-/is-path-cwd-1.0.0.tgz#d225ec23132e89edd38fda767472e62e65f1106d"
- integrity sha1-0iXsIxMuie3Tj9p2dHLmLmXxEG0=
+is-obj@^2.0.0:
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/is-obj/-/is-obj-2.0.0.tgz#473fb05d973705e3fd9620545018ca8e22ef4982"
+ integrity sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==
-is-path-in-cwd@^1.0.0:
- version "1.0.1"
- resolved "https://registry.yarnpkg.com/is-path-in-cwd/-/is-path-in-cwd-1.0.1.tgz#5ac48b345ef675339bd6c7a48a912110b241cf52"
- integrity sha512-FjV1RTW48E7CWM7eE/J2NJvAEEVektecDBVBE5Hh3nM1Jd0kvhHtX68Pr3xsDf857xt3Y4AkwVULK1Vku62aaQ==
+is-path-cwd@^2.0.0:
+ version "2.2.0"
+ resolved "https://registry.yarnpkg.com/is-path-cwd/-/is-path-cwd-2.2.0.tgz#67d43b82664a7b5191fd9119127eb300048a9fdb"
+ integrity sha512-w942bTcih8fdJPJmQHFzkS76NEP8Kzzvmw92cXsazb8intwLqPibPPdXf4ANdKV3rYMuuQYGIWtvz9JilB3NFQ==
+
+is-path-in-cwd@^2.0.0:
+ version "2.1.0"
+ resolved "https://registry.yarnpkg.com/is-path-in-cwd/-/is-path-in-cwd-2.1.0.tgz#bfe2dca26c69f397265a4009963602935a053acb"
+ integrity sha512-rNocXHgipO+rvnP6dk3zI20RpOtrAM/kzbB258Uw5BWr3TpXi861yzjo16Dn4hUox07iw5AyeMLHWsujkjzvRQ==
dependencies:
- is-path-inside "^1.0.0"
+ is-path-inside "^2.1.0"
is-path-inside@^1.0.0:
version "1.0.1"
@@ -5614,6 +6753,23 @@
dependencies:
path-is-inside "^1.0.1"
+is-path-inside@^2.1.0:
+ version "2.1.0"
+ resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-2.1.0.tgz#7c9810587d659a40d27bcdb4d5616eab059494b2"
+ integrity sha512-wiyhTzfDWsvwAW53OBWF5zuvaOGlZ6PwYxAbPVDhpm+gM09xKQGjBq/8uYN12aDvMxnAnq3dxTyoSoRNmg5YFg==
+ dependencies:
+ path-is-inside "^1.0.2"
+
+is-path-inside@^3.0.1, is-path-inside@^3.0.2:
+ version "3.0.2"
+ resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.2.tgz#f5220fc82a3e233757291dddc9c5877f2a1f3017"
+ integrity sha512-/2UGPSgmtqwo1ktx8NDHjuPwZWmHhO+gj0f93EkhLB5RgW9RZevWYYlIkS6zePc6U2WpOdQYIwHe9YC4DWEBVg==
+
+is-plain-obj@^1.0.0, is-plain-obj@^1.1.0:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-1.1.0.tgz#71a50c8429dfca773c92a390a4a03b39fcd51d3e"
+ integrity sha1-caUMhCnfync8kqOQpKA7OfzVHT4=
+
is-plain-object@^2.0.1, is-plain-object@^2.0.3, is-plain-object@^2.0.4:
version "2.0.4"
resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-2.0.4.tgz#2c163b3fafb1b606d9d17928f05c2a1c38e07677"
@@ -5626,6 +6782,14 @@
resolved "https://registry.yarnpkg.com/is-promise/-/is-promise-2.1.0.tgz#79a2a9ece7f096e80f36d2b2f3bc16c1ff4bf3fa"
integrity sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=
+is-proto-prop@^2.0.0:
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/is-proto-prop/-/is-proto-prop-2.0.0.tgz#99ab2863462e44090fd083efd1929058f9d935e1"
+ integrity sha512-jl3NbQ/fGLv5Jhan4uX+Ge9ohnemqyblWVVCpAvtTQzNFvV2xhJq+esnkIbYQ9F1nITXoLfDDQLp7LBw/zzncg==
+ dependencies:
+ lowercase-keys "^1.0.0"
+ proto-props "^2.0.0"
+
is-regex@^1.0.4, is-regex@^1.0.5:
version "1.0.5"
resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.0.5.tgz#39d589a358bf18967f726967120b8fc1aed74eae"
@@ -5638,6 +6802,13 @@
resolved "https://registry.yarnpkg.com/is-regexp/-/is-regexp-1.0.0.tgz#fd2d883545c46bac5a633e7b9a09e87fa2cb5069"
integrity sha1-/S2INUXEa6xaYz57mgnof6LLUGk=
+is-relative@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/is-relative/-/is-relative-1.0.0.tgz#a1bb6935ce8c5dba1e8b9754b9b2dcc020e2260d"
+ integrity sha512-Kw/ReK0iqwKeu0MITLFuj0jbPAmEiOsIwyIXvvbfa6QfmN9pkD1M+8pdk7Rl/dTKbH34/XBFMbgD4iMJhLQbGA==
+ dependencies:
+ is-unc-path "^1.0.0"
+
is-resolvable@^1.0.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/is-resolvable/-/is-resolvable-1.1.0.tgz#fb18f87ce1feb925169c9a407c19318a3206ed88"
@@ -5672,12 +6843,19 @@
dependencies:
has-symbols "^1.0.1"
-is-typedarray@~1.0.0:
+is-typedarray@^1.0.0, is-typedarray@~1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a"
integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=
-is-windows@^1.0.2:
+is-unc-path@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/is-unc-path/-/is-unc-path-1.0.0.tgz#d731e8898ed090a12c352ad2eaed5095ad322c9d"
+ integrity sha512-mrGpVd0fs7WWLfVsStvgF6iEJnbjDFZh9/emhRDcGWTduTfNHd9CHeUwH3gYIjdbwo4On6hunkztwOaAw0yllQ==
+ dependencies:
+ unc-path-regex "^0.1.2"
+
+is-windows@^1.0.1, is-windows@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/is-windows/-/is-windows-1.0.2.tgz#d1850eb9791ecd18e6182ce12a30f396634bb19d"
integrity sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==
@@ -5687,6 +6865,13 @@
resolved "https://registry.yarnpkg.com/is-wsl/-/is-wsl-1.1.0.tgz#1f16e4aa22b04d1336b66188a66af3c600c3a66d"
integrity sha1-HxbkqiKwTRM2tmGIpmrzxgDDpm0=
+is-wsl@^2.1.1:
+ version "2.2.0"
+ resolved "https://registry.yarnpkg.com/is-wsl/-/is-wsl-2.2.0.tgz#74a4c76e77ca9fd3f932f290c17ea326cd157271"
+ integrity sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==
+ dependencies:
+ is-docker "^2.0.0"
+
is-yarn-global@^0.3.0:
version "0.3.0"
resolved "https://registry.yarnpkg.com/is-yarn-global/-/is-yarn-global-0.3.0.tgz#d502d3382590ea3004893746754c89139973e232"
@@ -5786,7 +6971,7 @@
execa "^1.0.0"
throat "^4.0.0"
-jest-cli@^24.7.1:
+jest-cli@^24.9.0:
version "24.9.0"
resolved "https://registry.yarnpkg.com/jest-cli/-/jest-cli-24.9.0.tgz#ad2de62d07472d419c6abc301fc432b98b10d2af"
integrity sha512-+VLRKyitT3BWoMeSUIHRxV/2g8y9gw91Jh5z2UmXZzkZKpbC08CSehVxgHUwTpy+HwGcns/tqafQDJW7imYvGg==
@@ -5856,14 +7041,17 @@
jest-util "^24.9.0"
pretty-format "^24.9.0"
-jest-environment-jsdom-fourteen@0.1.0:
- version "0.1.0"
- resolved "https://registry.yarnpkg.com/jest-environment-jsdom-fourteen/-/jest-environment-jsdom-fourteen-0.1.0.tgz#aad6393a9d4b565b69a609109bf469f62bf18ccc"
- integrity sha512-4vtoRMg7jAstitRzL4nbw83VmGH8Rs13wrND3Ud2o1fczDhMUF32iIrNKwYGgeOPUdfvZU4oy8Bbv+ni1fgVCA==
+jest-environment-jsdom-fourteen@1.0.1:
+ version "1.0.1"
+ resolved "https://registry.yarnpkg.com/jest-environment-jsdom-fourteen/-/jest-environment-jsdom-fourteen-1.0.1.tgz#4cd0042f58b4ab666950d96532ecb2fc188f96fb"
+ integrity sha512-DojMX1sY+at5Ep+O9yME34CdidZnO3/zfPh8UW+918C5fIZET5vCjfkegixmsi7AtdYfkr4bPlIzmWnlvQkP7Q==
dependencies:
- jest-mock "^24.5.0"
- jest-util "^24.5.0"
- jsdom "^14.0.0"
+ "@jest/environment" "^24.3.0"
+ "@jest/fake-timers" "^24.3.0"
+ "@jest/types" "^24.3.0"
+ jest-mock "^24.0.0"
+ jest-util "^24.0.0"
+ jsdom "^14.1.0"
jest-environment-jsdom@^24.9.0:
version "24.9.0"
@@ -5966,7 +7154,7 @@
slash "^2.0.0"
stack-utils "^1.0.1"
-jest-mock@^24.5.0, jest-mock@^24.9.0:
+jest-mock@^24.0.0, jest-mock@^24.9.0:
version "24.9.0"
resolved "https://registry.yarnpkg.com/jest-mock/-/jest-mock-24.9.0.tgz#c22835541ee379b908673ad51087a2185c13f1c6"
integrity sha512-3BEYN5WbSq9wd+SyLDES7AHnjH9A/ROBwmz7l2y+ol+NtSFO8DYiEBzoO1CeFc9a8DYy10EO4dDFVv/wN3zl1w==
@@ -5992,18 +7180,7 @@
jest-regex-util "^24.3.0"
jest-snapshot "^24.9.0"
-jest-resolve@24.7.1:
- version "24.7.1"
- resolved "https://registry.yarnpkg.com/jest-resolve/-/jest-resolve-24.7.1.tgz#e4150198299298380a75a9fd55043fa3b9b17fde"
- integrity sha512-Bgrc+/UUZpGJ4323sQyj85hV9d+ANyPNu6XfRDUcyFNX1QrZpSoM0kE4Mb2vZMAYTJZsBFzYe8X1UaOkOELSbw==
- dependencies:
- "@jest/types" "^24.7.0"
- browser-resolve "^1.11.3"
- chalk "^2.0.1"
- jest-pnp-resolver "^1.2.1"
- realpath-native "^1.1.0"
-
-jest-resolve@^24.9.0:
+jest-resolve@24.9.0, jest-resolve@^24.9.0:
version "24.9.0"
resolved "https://registry.yarnpkg.com/jest-resolve/-/jest-resolve-24.9.0.tgz#dff04c7687af34c4dd7e524892d9cf77e5d17321"
integrity sha512-TaLeLVL1l08YFZAt3zaPtjiVvyy4oSA6CRe+0AFPPVX3Q/VI0giIWWoAvoS5L96vj9Dqxj4fB5p2qrHCmTU/MQ==
@@ -6092,7 +7269,7 @@
pretty-format "^24.9.0"
semver "^6.2.0"
-jest-util@^24.5.0, jest-util@^24.9.0:
+jest-util@^24.0.0, jest-util@^24.9.0:
version "24.9.0"
resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-24.9.0.tgz#7396814e48536d2e85a37de3e4c431d7cb140162"
integrity sha512-x+cZU8VRmOJxbA1K5oDBdxQmdq0OIdADarLxk0Mq+3XS4jgvhG/oKGWcIDCtPG0HgjxOYvF+ilPJQsAyXfbNOg==
@@ -6122,16 +7299,17 @@
leven "^3.1.0"
pretty-format "^24.9.0"
-jest-watch-typeahead@0.3.0:
- version "0.3.0"
- resolved "https://registry.yarnpkg.com/jest-watch-typeahead/-/jest-watch-typeahead-0.3.0.tgz#f56d9ee17ea71ecbf8253fed213df3185a1584c9"
- integrity sha512-+uOtlppt9ysST6k6ZTqsPI0WNz2HLa8bowiZylZoQCQaAVn7XsVmHhZREkz73FhKelrFrpne4hQQjdq42nFEmA==
+jest-watch-typeahead@0.4.2:
+ version "0.4.2"
+ resolved "https://registry.yarnpkg.com/jest-watch-typeahead/-/jest-watch-typeahead-0.4.2.tgz#e5be959698a7fa2302229a5082c488c3c8780a4a"
+ integrity sha512-f7VpLebTdaXs81rg/oj4Vg/ObZy2QtGzAmGLNsqUS5G5KtSN68tFcIsbvNODfNyQxU78g7D8x77o3bgfBTR+2Q==
dependencies:
- ansi-escapes "^3.0.0"
+ ansi-escapes "^4.2.1"
chalk "^2.4.1"
+ jest-regex-util "^24.9.0"
jest-watcher "^24.3.0"
- slash "^2.0.0"
- string-length "^2.0.0"
+ slash "^3.0.0"
+ string-length "^3.1.0"
strip-ansi "^5.0.0"
jest-watcher@^24.3.0, jest-watcher@^24.9.0:
@@ -6155,24 +7333,27 @@
merge-stream "^2.0.0"
supports-color "^6.1.0"
-jest@24.7.1:
- version "24.7.1"
- resolved "https://registry.yarnpkg.com/jest/-/jest-24.7.1.tgz#0d94331cf510c75893ee32f87d7321d5bf8f2501"
- integrity sha512-AbvRar5r++izmqo5gdbAjTeA6uNRGoNRuj5vHB0OnDXo2DXWZJVuaObiGgtlvhKb+cWy2oYbQSfxv7Q7GjnAtA==
+jest-worker@^25.1.0:
+ version "25.5.0"
+ resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-25.5.0.tgz#2611d071b79cea0f43ee57a3d118593ac1547db1"
+ integrity sha512-/dsSmUkIy5EBGfv/IjjqmFxrNAUpBERfGs1oHROyD7yxjG/w+t0GOJDX8O1k32ySmd7+a5IhnJU2qQFcJ4n1vw==
+ dependencies:
+ merge-stream "^2.0.0"
+ supports-color "^7.0.0"
+
+jest@24.9.0:
+ version "24.9.0"
+ resolved "https://registry.yarnpkg.com/jest/-/jest-24.9.0.tgz#987d290c05a08b52c56188c1002e368edb007171"
+ integrity sha512-YvkBL1Zm7d2B1+h5fHEOdyjCG+sGMz4f8D86/0HiqJ6MB4MnDc8FgP5vdWsGnemOQro7lnYo8UakZ3+5A0jxGw==
dependencies:
import-local "^2.0.0"
- jest-cli "^24.7.1"
+ jest-cli "^24.9.0"
jju@^1.1.0:
version "1.4.0"
resolved "https://registry.yarnpkg.com/jju/-/jju-1.4.0.tgz#a3abe2718af241a2b2904f84a625970f389ae32a"
integrity sha1-o6vicYryQaKykE+EpiWXDzia4yo=
-js-levenshtein@^1.1.3:
- version "1.1.6"
- resolved "https://registry.yarnpkg.com/js-levenshtein/-/js-levenshtein-1.1.6.tgz#c6cee58eb3550372df8deb85fad5ce66ce01d59d"
- integrity sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g==
-
"js-tokens@^3.0.0 || ^4.0.0", js-tokens@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499"
@@ -6183,7 +7364,12 @@
resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-3.0.2.tgz#9866df395102130e38f7f996bceb65443209c25b"
integrity sha1-mGbfOVECEw449/mWvOtlRDIJwls=
-js-yaml@^3.13.0, js-yaml@^3.13.1:
+js-types@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/js-types/-/js-types-1.0.0.tgz#d242e6494ed572ad3c92809fc8bed7f7687cbf03"
+ integrity sha1-0kLmSU7Vcq08koCfyL7X92h8vwM=
+
+js-yaml@^3.13.1:
version "3.13.1"
resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.13.1.tgz#aff151b30bfdfa8e49e05da22e7415e9dfa37847"
integrity sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==
@@ -6228,7 +7414,7 @@
ws "^5.2.0"
xml-name-validator "^3.0.0"
-jsdom@^14.0.0:
+jsdom@^14.1.0:
version "14.1.0"
resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-14.1.0.tgz#916463b6094956b0a6c1782c94e380cd30e1981b"
integrity sha512-O901mfJSuTdwU2w3Sn+74T+RnDVP+FuV5fH8tcPWyqrseRAb0s5xOtPgCFiPOtLcyK7CLIJwPyD83ZqQWvA5ng==
@@ -6368,6 +7554,13 @@
dependencies:
minimist "^1.2.0"
+json5@^2.1.1, json5@^2.1.2:
+ version "2.1.3"
+ resolved "https://registry.yarnpkg.com/json5/-/json5-2.1.3.tgz#c9b0f7fa9233bfe5807fe66fcf3a5617ed597d43"
+ integrity sha512-KXPvOm8K9IJKFM0bmdn8QXh7udDh1g/giieX0NLCaMnb4hEiVFqnop2ImTXCc5e0/oHz3LTqmHGtExn5hfMkOA==
+ dependencies:
+ minimist "^1.2.5"
+
jsonfile@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-4.0.0.tgz#8771aae0799b64076b76640fca058f9c10e33ecb"
@@ -6375,6 +7568,15 @@
optionalDependencies:
graceful-fs "^4.1.6"
+jsonfile@^6.0.1:
+ version "6.0.1"
+ resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.0.1.tgz#98966cba214378c8c84b82e085907b40bf614179"
+ integrity sha512-jR2b5v7d2vIOust+w3wtFKZIfpC2pnRmFAhAC/BuweZFQR8qZzxH1OyrQ10HmdVYiXWkYUqPVsz91cG7EL2FBg==
+ dependencies:
+ universalify "^1.0.0"
+ optionalDependencies:
+ graceful-fs "^4.1.6"
+
jsonify@~0.0.0:
version "0.0.0"
resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.0.tgz#2c74b6ee41d93ca51b7b5aaee8f503631d252a73"
@@ -6390,7 +7592,7 @@
json-schema "0.2.3"
verror "1.10.0"
-jsx-ast-utils@^2.0.1:
+jsx-ast-utils@^2.2.1, jsx-ast-utils@^2.2.3:
version "2.2.3"
resolved "https://registry.yarnpkg.com/jsx-ast-utils/-/jsx-ast-utils-2.2.3.tgz#8a9364e402448a3ce7f14d357738310d9248054f"
integrity sha512-EdIHFMm+1BPynpKOpdPqiOsvnIrInRGJD7bzPZdPkjitQEqpdpUuFpq4T0npZFKTiB3RhWFdGN+oqOJIdhDhQA==
@@ -6405,7 +7607,7 @@
dependencies:
json-buffer "3.0.0"
-killable@^1.0.0:
+killable@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/killable/-/killable-1.0.1.tgz#4c8ce441187a061c7474fb87ca08e2a638194892"
integrity sha512-LzqtLKlUwirEUyl/nicirVmNiPvYs7l5n8wOPP7fyJVpUPkvCnW/vuiXGpylGUlnPDnB7311rARzAt3Mhswpjg==
@@ -6513,6 +7715,13 @@
resolved "https://registry.yarnpkg.com/leven/-/leven-3.1.0.tgz#77891de834064cccba82ae7842bb6b14a13ed7f2"
integrity sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==
+levenary@^1.1.1:
+ version "1.1.1"
+ resolved "https://registry.yarnpkg.com/levenary/-/levenary-1.1.1.tgz#842a9ee98d2075aa7faeedbe32679e9205f46f77"
+ integrity sha512-mkAdOIt79FD6irqjYSs4rdbnlT5vRonMEvBVPVb3XmevfS8kgRXwfes0dhPdEtzTWD/1eNE/Bm/G1iRt6DcnQQ==
+ dependencies:
+ leven "^3.1.0"
+
levn@^0.3.0, levn@~0.3.0:
version "0.3.0"
resolved "https://registry.yarnpkg.com/levn/-/levn-0.3.0.tgz#3b09924edf9f083c0490fdd4c0bc4421e04764ee"
@@ -6521,6 +7730,18 @@
prelude-ls "~1.1.2"
type-check "~0.3.2"
+line-column-path@^2.0.0:
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/line-column-path/-/line-column-path-2.0.0.tgz#439aff48ef80d74c475801a25b560d021acf1288"
+ integrity sha512-nz3A+vi4bElhwd62E9+Qk/f9BDYLSzD/4Hy1rir0I4GnMxSTezSymzANyph5N1PgRZ3sSbA+yR5hOuXxc71a0Q==
+ dependencies:
+ type-fest "^0.4.1"
+
+lines-and-columns@^1.1.6:
+ version "1.1.6"
+ resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.1.6.tgz#1c00c743b433cd0a4e80758f7b64a57440d9ff00"
+ integrity sha1-HADHQ7QzzQpOgHWPe2SldEDZ/wA=
+
load-json-file@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-2.0.0.tgz#7947e42149af80d696cbf797bcaabcfe1fe29ca8"
@@ -6541,20 +7762,20 @@
pify "^3.0.0"
strip-bom "^3.0.0"
-loader-fs-cache@^1.0.0:
- version "1.0.2"
- resolved "https://registry.yarnpkg.com/loader-fs-cache/-/loader-fs-cache-1.0.2.tgz#54cedf6b727e1779fd8f01205f05f6e88706f086"
- integrity sha512-70IzT/0/L+M20jUlEqZhZyArTU6VKLRTYRDAYN26g4jfzpJqjipLL3/hgYpySqI9PwsVRHHFja0LfEmsx9X2Cw==
+loader-fs-cache@^1.0.2:
+ version "1.0.3"
+ resolved "https://registry.yarnpkg.com/loader-fs-cache/-/loader-fs-cache-1.0.3.tgz#f08657646d607078be2f0a032f8bd69dd6f277d9"
+ integrity sha512-ldcgZpjNJj71n+2Mf6yetz+c9bM4xpKtNds4LbqXzU/PTdeAX0g3ytnU1AJMEcTk2Lex4Smpe3Q/eCTsvUBxbA==
dependencies:
find-cache-dir "^0.1.1"
- mkdirp "0.5.1"
+ mkdirp "^0.5.1"
-loader-runner@^2.3.0:
+loader-runner@^2.4.0:
version "2.4.0"
resolved "https://registry.yarnpkg.com/loader-runner/-/loader-runner-2.4.0.tgz#ed47066bfe534d7e84c4c7b9998c2a75607d9357"
integrity sha512-Jsmr89RcXGIwivFY21FcRrisYZfvLMTWx5kOLc+JTxtpBOG6xML0vzbc6SEQG2FO9/4Fc3wW4LVcB5DmGflaRw==
-loader-utils@1.2.3, loader-utils@^1.0.1, loader-utils@^1.0.2, loader-utils@^1.1.0, loader-utils@^1.2.3:
+loader-utils@1.2.3, loader-utils@^1.1.0, loader-utils@^1.2.3:
version "1.2.3"
resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-1.2.3.tgz#1ff5dc6911c9f0a062531a4c04b609406108c2c7"
integrity sha512-fkpz8ejdnEMG3s37wGL07iSBDg99O9D5yflE9RGNH3hRdx9SOwYfnGYdZOUIZitN8E+E2vkq3MUMYMvPYl5ZZA==
@@ -6563,6 +7784,15 @@
emojis-list "^2.0.0"
json5 "^1.0.1"
+loader-utils@^1.4.0:
+ version "1.4.0"
+ resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-1.4.0.tgz#c579b5e34cb34b1a74edc6c1fb36bfa371d5a613"
+ integrity sha512-qH0WSMBtn/oHuwjy/NucEgbx5dbxxnxup9s4PVXJUDHZBQY+s0NWA9rJf53RBnQZxfch7euUui7hpoAPvALZdA==
+ dependencies:
+ big.js "^5.2.2"
+ emojis-list "^3.0.0"
+ json5 "^1.0.1"
+
locate-path@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-2.0.0.tgz#2b568b265eec944c6d9c0de9c3dbbbca0354cd8e"
@@ -6579,6 +7809,13 @@
p-locate "^3.0.0"
path-exists "^3.0.0"
+locate-path@^5.0.0:
+ version "5.0.0"
+ resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-5.0.0.tgz#1afba396afd676a6d42504d0a67a3a7eb9f62aa0"
+ integrity sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==
+ dependencies:
+ p-locate "^4.1.0"
+
lodash-id@^0.14.0:
version "0.14.0"
resolved "https://registry.yarnpkg.com/lodash-id/-/lodash-id-0.14.0.tgz#baf48934e543a1b5d6346f8c84698b1a8c803896"
@@ -6599,6 +7836,11 @@
resolved "https://registry.yarnpkg.com/lodash.flow/-/lodash.flow-3.5.0.tgz#87bf40292b8cf83e4e8ce1a3ae4209e20071675a"
integrity sha1-h79AKSuM+D5OjOGjrkIJ4gBxZ1o=
+lodash.get@^4.4.2:
+ version "4.4.2"
+ resolved "https://registry.yarnpkg.com/lodash.get/-/lodash.get-4.4.2.tgz#2d177f652fa31e939b4438d5341499dfa3825e99"
+ integrity sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=
+
lodash.memoize@^4.1.2:
version "4.1.2"
resolved "https://registry.yarnpkg.com/lodash.memoize/-/lodash.memoize-4.1.2.tgz#bcc6c49a42a2840ed997f323eada5ecd182e0bfe"
@@ -6609,11 +7851,6 @@
resolved "https://registry.yarnpkg.com/lodash.sortby/-/lodash.sortby-4.7.0.tgz#edd14c824e2cc9c1e0b0a1b42bb5210516a42438"
integrity sha1-7dFMgk4sycHgsKG0K7UhBRakJDg=
-lodash.tail@^4.1.1:
- version "4.1.1"
- resolved "https://registry.yarnpkg.com/lodash.tail/-/lodash.tail-4.1.1.tgz#d2333a36d9e7717c8ad2f7cacafec7c32b444664"
- integrity sha1-0jM6NtnncXyK0vfKyv7HwytERmQ=
-
lodash.template@^4.4.0, lodash.template@^4.5.0:
version "4.5.0"
resolved "https://registry.yarnpkg.com/lodash.template/-/lodash.template-4.5.0.tgz#f976195cf3f347d0d5f52483569fe8031ccce8ab"
@@ -6634,25 +7871,32 @@
resolved "https://registry.yarnpkg.com/lodash.throttle/-/lodash.throttle-4.1.1.tgz#c23e91b710242ac70c37f1e1cda9274cc39bf2f4"
integrity sha1-wj6RtxAkKscMN/HhzaknTMOb8vQ=
-lodash.unescape@4.0.1:
- version "4.0.1"
- resolved "https://registry.yarnpkg.com/lodash.unescape/-/lodash.unescape-4.0.1.tgz#bf2249886ce514cda112fae9218cdc065211fc9c"
- integrity sha1-vyJJiGzlFM2hEvrpIYzcBlIR/Jw=
-
lodash.uniq@^4.5.0:
version "4.5.0"
resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773"
integrity sha1-0CJTc662Uq3BvILklFM5qEJ1R3M=
-lodash@4, "lodash@>=3.5 <5", lodash@^4.16.5, lodash@^4.17.10, lodash@^4.17.11, lodash@^4.17.12, lodash@^4.17.13, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.4, lodash@^4.17.5:
+lodash.zip@^4.2.0:
+ version "4.2.0"
+ resolved "https://registry.yarnpkg.com/lodash.zip/-/lodash.zip-4.2.0.tgz#ec6662e4896408ed4ab6c542a3990b72cc080020"
+ integrity sha1-7GZi5IlkCO1KtsVCo5kLcswIACA=
+
+lodash@4, "lodash@>=3.5 <5", lodash@^4.13.1, lodash@^4.16.5, lodash@^4.17.11, lodash@^4.17.13, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.4, lodash@^4.17.5:
version "4.17.15"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.15.tgz#b447f6670a0455bbfeedd11392eff330ea097548"
integrity sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==
-loglevel@^1.4.1:
- version "1.6.6"
- resolved "https://registry.yarnpkg.com/loglevel/-/loglevel-1.6.6.tgz#0ee6300cc058db6b3551fa1c4bf73b83bb771312"
- integrity sha512-Sgr5lbboAUBo3eXCSPL4/KoVz3ROKquOjcctxmHIt+vol2DrqTQe3SwkKKuYhEiWB5kYa13YyopJ69deJ1irzQ==
+log-symbols@^3.0.0:
+ version "3.0.0"
+ resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-3.0.0.tgz#f3a08516a5dea893336a7dee14d18a1cfdab77c4"
+ integrity sha512-dSkNGuI7iG3mfvDzUuYZyvk5dD9ocYCYzNU6CYDE6+Xqd+gwme6Z00NS3dUh8mq/73HaEtT7m6W+yUPtU6BZnQ==
+ dependencies:
+ chalk "^2.4.2"
+
+loglevel@^1.6.6:
+ version "1.6.8"
+ resolved "https://registry.yarnpkg.com/loglevel/-/loglevel-1.6.8.tgz#8a25fb75d092230ecd4457270d80b54e28011171"
+ integrity sha512-bsU7+gc9AJ2SqpzxwU3+1fedl8zAntbtC5XYlt3s2j1hJcn2PsXSmgN8TaLG/J1/2mod4+cE/3vNL70/c1RNCA==
loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.2.0, loose-envify@^1.3.1, loose-envify@^1.4.0:
version "1.4.0"
@@ -6661,6 +7905,14 @@
dependencies:
js-tokens "^3.0.0 || ^4.0.0"
+loud-rejection@^1.0.0:
+ version "1.6.0"
+ resolved "https://registry.yarnpkg.com/loud-rejection/-/loud-rejection-1.6.0.tgz#5b46f80147edee578870f086d04821cf998e551f"
+ integrity sha1-W0b4AUft7leIcPCG0Eghz5mOVR8=
+ dependencies:
+ currently-unhandled "^0.4.1"
+ signal-exit "^3.0.0"
+
lowdb@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/lowdb/-/lowdb-1.0.0.tgz#5243be6b22786ccce30e50c9a33eac36b20c8064"
@@ -6672,10 +7924,12 @@
pify "^3.0.0"
steno "^0.4.1"
-lower-case@^1.1.1:
- version "1.1.4"
- resolved "https://registry.yarnpkg.com/lower-case/-/lower-case-1.1.4.tgz#9a2cabd1b9e8e0ae993a4bf7d5875c39c42e8eac"
- integrity sha1-miyr0bno4K6ZOkv31YdcOcQujqw=
+lower-case@^2.0.1:
+ version "2.0.1"
+ resolved "https://registry.yarnpkg.com/lower-case/-/lower-case-2.0.1.tgz#39eeb36e396115cc05e29422eaea9e692c9408c7"
+ integrity sha512-LiWgfDLLb1dwbFQZsSglpRj+1ctGnayXz3Uv0/WO8n558JycT5fg6zkNcnW0G68Nn0aEldTFeEfmjCfmqry/rQ==
+ dependencies:
+ tslib "^1.10.0"
lowercase-keys@^1.0.0, lowercase-keys@^1.0.1:
version "1.0.1"
@@ -6717,6 +7971,13 @@
pify "^4.0.1"
semver "^5.6.0"
+make-dir@^3.0.0, make-dir@^3.0.2:
+ version "3.1.0"
+ resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f"
+ integrity sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==
+ dependencies:
+ semver "^6.0.0"
+
makeerror@1.0.x:
version "1.0.11"
resolved "https://registry.yarnpkg.com/makeerror/-/makeerror-1.0.11.tgz#e01a5c9109f2af79660e4e8b9587790184f5a96c"
@@ -6741,6 +8002,16 @@
resolved "https://registry.yarnpkg.com/map-cache/-/map-cache-0.2.2.tgz#c32abd0bd6525d9b051645bb4f26ac5dc98a0dbf"
integrity sha1-wyq9C9ZSXZsFFkW7TyasXcmKDb8=
+map-obj@^1.0.0:
+ version "1.0.1"
+ resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-1.0.1.tgz#d933ceb9205d82bdcf4886f6742bdc2b4dea146d"
+ integrity sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0=
+
+map-obj@^2.0.0:
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-2.0.0.tgz#a65cd29087a92598b8791257a523e021222ac1f9"
+ integrity sha1-plzSkIepJZi4eRJXpSPgISIqwfk=
+
map-visit@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/map-visit/-/map-visit-1.0.0.tgz#ecdca8f13144e660f1b5bd41f12f3479d98dfb8f"
@@ -6776,7 +8047,12 @@
mimic-fn "^2.0.0"
p-is-promise "^2.0.0"
-memory-fs@^0.4.1, memory-fs@~0.4.1:
+memory-fs@^0.2.0:
+ version "0.2.0"
+ resolved "https://registry.yarnpkg.com/memory-fs/-/memory-fs-0.2.0.tgz#f2bb25368bc121e391c2520de92969caee0a0290"
+ integrity sha1-8rslNovBIeORwlIN6Slpyu4KApA=
+
+memory-fs@^0.4.1:
version "0.4.1"
resolved "https://registry.yarnpkg.com/memory-fs/-/memory-fs-0.4.1.tgz#3a9a20b8462523e447cfbc7e8bb80ed667bfc552"
integrity sha1-OpoguEYlI+RHz7x+i7gO1me/xVI=
@@ -6797,6 +8073,21 @@
resolved "https://registry.yarnpkg.com/memorystream/-/memorystream-0.3.1.tgz#86d7090b30ce455d63fbae12dda51a47ddcaf9b2"
integrity sha1-htcJCzDORV1j+64S3aUaR93K+bI=
+meow@^5.0.0:
+ version "5.0.0"
+ resolved "https://registry.yarnpkg.com/meow/-/meow-5.0.0.tgz#dfc73d63a9afc714a5e371760eb5c88b91078aa4"
+ integrity sha512-CbTqYU17ABaLefO8vCU153ZZlprKYWDljcndKKDCFcYQITzWCXZAVk4QMFZPgvzrnUQ3uItnIE/LoUOwrT15Ig==
+ dependencies:
+ camelcase-keys "^4.0.0"
+ decamelize-keys "^1.0.0"
+ loud-rejection "^1.0.0"
+ minimist-options "^3.0.1"
+ normalize-package-data "^2.3.4"
+ read-pkg-up "^3.0.0"
+ redent "^2.0.0"
+ trim-newlines "^2.0.0"
+ yargs-parser "^10.0.0"
+
merge-deep@^3.0.2:
version "3.0.2"
resolved "https://registry.yarnpkg.com/merge-deep/-/merge-deep-3.0.2.tgz#f39fa100a4f1bd34ff29f7d2bf4508fbb8d83ad2"
@@ -6836,12 +8127,17 @@
resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee"
integrity sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=
+micro-spelling-correcter@^1.1.1:
+ version "1.1.1"
+ resolved "https://registry.yarnpkg.com/micro-spelling-correcter/-/micro-spelling-correcter-1.1.1.tgz#805a06a26ccfcad8f3e5c6a1ac5ff29d4530166e"
+ integrity sha512-lkJ3Rj/mtjlRcHk6YyCbvZhyWTOzdBvTHsxMmZSk5jxN1YyVSQ+JETAom55mdzfcyDrY/49Z7UCW760BK30crg==
+
microevent.ts@~0.1.1:
version "0.1.1"
resolved "https://registry.yarnpkg.com/microevent.ts/-/microevent.ts-0.1.1.tgz#70b09b83f43df5172d0205a63025bce0f7357fa0"
integrity sha512-jo1OfR4TaEwd5HOrt5+tAZ9mqT4jmpNAusXtyfNzqVm9uiSYFZlKM1wYL4oU7azZW/PxQW53wM0S6OR1JHNa2g==
-micromatch@^3.1.10, micromatch@^3.1.4, micromatch@^3.1.8:
+micromatch@^3.1.10, micromatch@^3.1.4:
version "3.1.10"
resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-3.1.10.tgz#70859bc95c9840952f359a068a3fc49f9ecfac23"
integrity sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==
@@ -6860,6 +8156,14 @@
snapdragon "^0.8.1"
to-regex "^3.0.2"
+micromatch@^4.0.2:
+ version "4.0.2"
+ resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.2.tgz#4fcb0999bf9fbc2fcbdd212f6d629b9a56c39259"
+ integrity sha512-y7FpHSbMUMoyPbYUSzO6PaZ6FyRnQOpHuKwbo1G+Knck95XVU4QAiKdGEnj5wwoS7PlOgthX/09u5iFJ+aYf5Q==
+ dependencies:
+ braces "^3.0.1"
+ picomatch "^2.0.5"
+
miller-rabin@^4.0.0:
version "4.0.1"
resolved "https://registry.yarnpkg.com/miller-rabin/-/miller-rabin-4.0.1.tgz#f080351c865b0dc562a8462966daa53543c78a4d"
@@ -6885,17 +8189,12 @@
resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1"
integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==
-mime@^2.0.3, mime@^2.4.4:
+mime@^2.4.4:
version "2.4.4"
resolved "https://registry.yarnpkg.com/mime/-/mime-2.4.4.tgz#bd7b91135fc6b01cde3e9bae33d659b63d8857e5"
integrity sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA==
-mimic-fn@^1.0.0:
- version "1.2.0"
- resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-1.2.0.tgz#820c86a39334640e99516928bd03fca88057d022"
- integrity sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==
-
-mimic-fn@^2.0.0:
+mimic-fn@^2.0.0, mimic-fn@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b"
integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==
@@ -6914,12 +8213,13 @@
gud "^1.0.0"
tiny-warning "^1.0.2"
-mini-css-extract-plugin@0.5.0:
- version "0.5.0"
- resolved "https://registry.yarnpkg.com/mini-css-extract-plugin/-/mini-css-extract-plugin-0.5.0.tgz#ac0059b02b9692515a637115b0cc9fed3a35c7b0"
- integrity sha512-IuaLjruM0vMKhUUT51fQdQzBYTX49dLj8w68ALEAe2A4iYNpIC4eMac67mt3NzycvjOlf07/kYxJDc0RTl1Wqw==
+mini-css-extract-plugin@0.9.0:
+ version "0.9.0"
+ resolved "https://registry.yarnpkg.com/mini-css-extract-plugin/-/mini-css-extract-plugin-0.9.0.tgz#47f2cf07aa165ab35733b1fc97d4c46c0564339e"
+ integrity sha512-lp3GeY7ygcgAmVIcRPBVhIkf8Us7FZjA+ILpal44qLdSu11wmjKQ3d9k15lfD7pO4esu9eUIAW7qiYIBppv40A==
dependencies:
loader-utils "^1.1.0"
+ normalize-url "1.9.1"
schema-utils "^1.0.0"
webpack-sources "^1.1.0"
@@ -6950,6 +8250,14 @@
dependencies:
brace-expansion "^1.1.7"
+minimist-options@^3.0.1:
+ version "3.0.2"
+ resolved "https://registry.yarnpkg.com/minimist-options/-/minimist-options-3.0.2.tgz#fba4c8191339e13ecf4d61beb03f070103f3d954"
+ integrity sha512-FyBrT/d0d4+uiZRbqznPXqw3IpZZG3gl3wKWiX784FycUKVwBt0uLBFkQrtE4tZOrgo78nZp2jnKz3L65T5LdQ==
+ dependencies:
+ arrify "^1.0.1"
+ is-plain-obj "^1.1.0"
+
minimist@0.0.8:
version "0.0.8"
resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d"
@@ -6960,11 +8268,44 @@
resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284"
integrity sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=
+minimist@^1.2.5:
+ version "1.2.5"
+ resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602"
+ integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==
+
minimist@~0.0.1:
version "0.0.10"
resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.10.tgz#de3f98543dbf96082be48ad1a0c7cda836301dcf"
integrity sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8=
+minipass-collect@^1.0.2:
+ version "1.0.2"
+ resolved "https://registry.yarnpkg.com/minipass-collect/-/minipass-collect-1.0.2.tgz#22b813bf745dc6edba2576b940022ad6edc8c617"
+ integrity sha512-6T6lH0H8OG9kITm/Jm6tdooIbogG9e0tLgpY6mphXSm/A9u8Nq1ryBG+Qspiub9LjWlBPsPS3tWQ/Botq4FdxA==
+ dependencies:
+ minipass "^3.0.0"
+
+minipass-flush@^1.0.5:
+ version "1.0.5"
+ resolved "https://registry.yarnpkg.com/minipass-flush/-/minipass-flush-1.0.5.tgz#82e7135d7e89a50ffe64610a787953c4c4cbb373"
+ integrity sha512-JmQSYYpPUqX5Jyn1mXaRwOda1uQ8HP5KAT/oDSLCzt1BYRhQU0/hDtsB1ufZfEEzMZ9aAVmsBw8+FWsIXlClWw==
+ dependencies:
+ minipass "^3.0.0"
+
+minipass-pipeline@^1.2.2:
+ version "1.2.2"
+ resolved "https://registry.yarnpkg.com/minipass-pipeline/-/minipass-pipeline-1.2.2.tgz#3dcb6bb4a546e32969c7ad710f2c79a86abba93a"
+ integrity sha512-3JS5A2DKhD2g0Gg8x3yamO0pj7YeKGwVlDS90pF++kxptwx/F+B//roxf9SqYil5tQo65bijy+dAuAFZmYOouA==
+ dependencies:
+ minipass "^3.0.0"
+
+minipass@^3.0.0, minipass@^3.1.1:
+ version "3.1.1"
+ resolved "https://registry.yarnpkg.com/minipass/-/minipass-3.1.1.tgz#7607ce778472a185ad6d89082aa2070f79cedcd5"
+ integrity sha512-UFqVihv6PQgwj8/yTGvl9kPz7xIAY+R5z6XYjRInD3Gk3qx6QGSD6zEcpeG4Dy/lQnv1J6zv8ejV90hyYIKf3w==
+ dependencies:
+ yallist "^4.0.0"
+
mississippi@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/mississippi/-/mississippi-3.0.0.tgz#ea0a3291f97e0b5e8776b363d5f0a12d94c67022"
@@ -6997,13 +8338,20 @@
for-in "^0.1.3"
is-extendable "^0.1.1"
-mkdirp@0.5.1, mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.0, mkdirp@~0.5.1:
+mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.1:
version "0.5.1"
resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903"
integrity sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=
dependencies:
minimist "0.0.8"
+mkdirp@^0.5.3:
+ version "0.5.5"
+ resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def"
+ integrity sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==
+ dependencies:
+ minimist "^1.2.5"
+
moment@2.x, moment@^2.24.0:
version "2.24.0"
resolved "https://registry.yarnpkg.com/moment/-/moment-2.24.0.tgz#0d055d53f5052aa653c9f6eb68bb5d12bf5c2b5b"
@@ -7060,15 +8408,20 @@
dns-packet "^1.3.1"
thunky "^1.0.2"
+multimap@^1.0.2:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/multimap/-/multimap-1.1.0.tgz#5263febc085a1791c33b59bb3afc6a76a2a10ca8"
+ integrity sha512-0ZIR9PasPxGXmRsEF8jsDzndzHDj7tIav+JUmvIFB/WHswliFnquxECT/De7GR4yg99ky/NlRKJT82G1y271bw==
+
mutationobserver-shim@^0.3.2:
version "0.3.3"
resolved "https://registry.yarnpkg.com/mutationobserver-shim/-/mutationobserver-shim-0.3.3.tgz#65869630bc89d7bf8c9cd9cb82188cd955aacd2b"
integrity sha512-gciOLNN8Vsf7YzcqRjKzlAJ6y7e+B86u7i3KXes0xfxx/nfLmozlW1Vn+Sc9x3tPIePFgc1AeIFhtRgkqTjzDQ==
-mute-stream@0.0.7:
- version "0.0.7"
- resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.7.tgz#3075ce93bc21b8fab43e1bc4da7e8115ed1e7bab"
- integrity sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=
+mute-stream@0.0.8:
+ version "0.0.8"
+ resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.8.tgz#1630c42b2251ff81e2a283de96a5497ea92e5e0d"
+ integrity sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==
nan@^2.12.1:
version "2.14.0"
@@ -7107,22 +8460,28 @@
resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb"
integrity sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==
-neo-async@^2.5.0, neo-async@^2.6.0:
+neo-async@^2.5.0, neo-async@^2.6.0, neo-async@^2.6.1:
version "2.6.1"
resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.1.tgz#ac27ada66167fa8849a6addd837f6b189ad2081c"
integrity sha512-iyam8fBuCUpWeKPGpaNMetEocMt364qkCsfL9JuhjXX6dRnguRVOfk2GZaDpPjcOKiiXCPINZC1GczQ7iTq3Zw==
+next-tick@~1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.0.0.tgz#ca86d1fe8828169b0120208e3dc8424b9db8342c"
+ integrity sha1-yobR/ogoFpsBICCOPchCS524NCw=
+
nice-try@^1.0.4:
version "1.0.5"
resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366"
integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==
-no-case@^2.2.0:
- version "2.3.2"
- resolved "https://registry.yarnpkg.com/no-case/-/no-case-2.3.2.tgz#60b813396be39b3f1288a4c1ed5d1e7d28b464ac"
- integrity sha512-rmTZ9kz+f3rCvK2TD1Ue/oZlns7OGoIWP4fc3llxxRXlOkHKoWPPWJOfFYpITabSow43QJbRIoHQXtt10VldyQ==
+no-case@^3.0.3:
+ version "3.0.3"
+ resolved "https://registry.yarnpkg.com/no-case/-/no-case-3.0.3.tgz#c21b434c1ffe48b39087e86cfb4d2582e9df18f8"
+ integrity sha512-ehY/mVQCf9BL0gKfsJBvFJen+1V//U+0HQMPrWct40ixE4jnv0bfvxDbWtAHL9EcaPEOJHVVYKoQn1TlZUB8Tw==
dependencies:
- lower-case "^1.1.1"
+ lower-case "^2.0.1"
+ tslib "^1.10.0"
node-fetch@^1.0.1:
version "1.7.3"
@@ -7142,7 +8501,7 @@
resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b"
integrity sha1-h6kGXNs1XTGC2PlM4RGIuCXGijs=
-node-libs-browser@^2.0.0:
+"node-libs-browser@^1.0.0 || ^2.0.0", node-libs-browser@^2.2.1:
version "2.2.1"
resolved "https://registry.yarnpkg.com/node-libs-browser/-/node-libs-browser-2.2.1.tgz#b64f513d18338625f90346d27b0d235e631f6425"
integrity sha512-h/zcD8H9kaDZ9ALUWwlBUDo6TKF8a7qBSCSEGfjTVIYeqsioSKaAX+BN7NgiMGp6iSIXZ3PxgCu8KS3b71YK5Q==
@@ -7187,14 +8546,19 @@
shellwords "^0.1.1"
which "^1.3.0"
-node-releases@^1.1.29, node-releases@^1.1.44:
+node-releases@^1.1.44:
version "1.1.44"
resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-1.1.44.tgz#cd66438a6eb875e3eb012b6a12e48d9f4326ffd7"
integrity sha512-NwbdvJyR7nrcGrXvKAvzc5raj/NkoJudkarh2yIpJ4t0NH4aqjUDz/486P+ynIW5eokKOfzGNRdYoLfBlomruw==
dependencies:
semver "^6.3.0"
-normalize-package-data@^2.3.2:
+node-releases@^1.1.52, node-releases@^1.1.53:
+ version "1.1.55"
+ resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-1.1.55.tgz#8af23b7c561d8e2e6e36a46637bab84633b07cee"
+ integrity sha512-H3R3YR/8TjT5WPin/wOoHOUPHgvj8leuU/Keta/rwelEQN9pA/S2Dx8/se4pZ2LBxSd0nAGzsNzhqwa77v7F1w==
+
+normalize-package-data@^2.3.2, normalize-package-data@^2.3.4, normalize-package-data@^2.5.0:
version "2.5.0"
resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.5.0.tgz#e66db1838b200c1dfc233225d12cb36520e234a8"
integrity sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==
@@ -7211,7 +8575,7 @@
dependencies:
remove-trailing-separator "^1.0.1"
-normalize-path@^3.0.0:
+normalize-path@^3.0.0, normalize-path@~3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65"
integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==
@@ -7221,6 +8585,16 @@
resolved "https://registry.yarnpkg.com/normalize-range/-/normalize-range-0.1.2.tgz#2d10c06bdfd312ea9777695a4d28439456b75942"
integrity sha1-LRDAa9/TEuqXd2laTShDlFa3WUI=
+normalize-url@1.9.1:
+ version "1.9.1"
+ resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-1.9.1.tgz#2cc0d66b31ea23036458436e3620d85954c66c3c"
+ integrity sha1-LMDWazHqIwNkWENuNiDYWVTGbDw=
+ dependencies:
+ object-assign "^4.0.1"
+ prepend-http "^1.0.0"
+ query-string "^4.1.0"
+ sort-keys "^1.0.0"
+
normalize-url@^3.0.0:
version "3.3.0"
resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-3.3.0.tgz#b2e1c4dc4f7c6d57743df733a4f5978d18650559"
@@ -7280,6 +8654,11 @@
resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455"
integrity sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==
+obj-props@^1.0.0:
+ version "1.3.0"
+ resolved "https://registry.yarnpkg.com/obj-props/-/obj-props-1.3.0.tgz#8884ab21c8d8496c4a7f696c78bf82289c51680b"
+ integrity sha512-k2Xkjx5wn6eC3537SWAXHzB6lkI81kS+icMKMkh4nG3w7shWG6MaWOBrNvhWVOszrtL5uxdfymQQfPUxwY+2eg==
+
object-assign@4.x, object-assign@^4, object-assign@^4.0.1, object-assign@^4.1.0, object-assign@^4.1.1:
version "4.1.1"
resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863"
@@ -7294,10 +8673,10 @@
define-property "^0.2.5"
kind-of "^3.0.3"
-object-hash@^1.1.4:
- version "1.3.1"
- resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-1.3.1.tgz#fde452098a951cb145f039bb7d455449ddc126df"
- integrity sha512-OSuu/pU4ENM9kmREg0BdNrUDIl1heYa4mBZacJc+vVWz4GtAwu7jO8s4AIt2aGRUTqxykpWzI3Oqnsm13tTMDA==
+object-hash@^2.0.1:
+ version "2.0.3"
+ resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-2.0.3.tgz#d12db044e03cd2ca3d77c0570d87225b02e1e6ea"
+ integrity sha512-JPKn0GMu+Fa3zt3Bmr66JhokJU5BaNBIh4ZeTlaCBzrBsOeXzwcKKAK1tbLiPKgvwmPXsDvvLHoWh5Bm7ofIYg==
object-inspect@^1.7.0:
version "1.7.0"
@@ -7314,6 +8693,11 @@
resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.1.tgz#1c47f272df277f3b1daf061677d9c82e2322c60e"
integrity sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==
+object-path@0.11.4:
+ version "0.11.4"
+ resolved "https://registry.yarnpkg.com/object-path/-/object-path-0.11.4.tgz#370ae752fbf37de3ea70a861c23bba8915691949"
+ integrity sha1-NwrnUvvzfePqcKhhwju6iRVpGUk=
+
object-visit@^1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/object-visit/-/object-visit-1.0.1.tgz#f79c4493af0c5377b59fe39d395e41042dd045bb"
@@ -7331,7 +8715,17 @@
has-symbols "^1.0.0"
object-keys "^1.0.11"
-object.fromentries@^2.0.0:
+object.entries@^1.1.0, object.entries@^1.1.1:
+ version "1.1.1"
+ resolved "https://registry.yarnpkg.com/object.entries/-/object.entries-1.1.1.tgz#ee1cf04153de02bb093fec33683900f57ce5399b"
+ integrity sha512-ilqR7BgdyZetJutmDPfXCDffGa0/Yzl2ivVNpbx/g4UeWrCdRnFDUBrKJGLhGieRHDATnyZXWBeCb29k9CJysQ==
+ dependencies:
+ define-properties "^1.1.3"
+ es-abstract "^1.17.0-next.1"
+ function-bind "^1.1.1"
+ has "^1.0.3"
+
+object.fromentries@^2.0.2:
version "2.0.2"
resolved "https://registry.yarnpkg.com/object.fromentries/-/object.fromentries-2.0.2.tgz#4a09c9b9bb3843dd0f89acdb517a794d4f355ac9"
integrity sha512-r3ZiBH7MQppDJVLx6fhD618GKNG40CZYH9wgwdhKxBDDbQgjeWGGd4AtkZad84d291YxvWe7bJGuE65Anh0dxQ==
@@ -7356,7 +8750,7 @@
dependencies:
isobject "^3.0.1"
-object.values@^1.1.0:
+object.values@^1.1.0, object.values@^1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/object.values/-/object.values-1.1.1.tgz#68a99ecde356b7e9295a3c5e0ce31dc8c953de5e"
integrity sha512-WTa54g2K8iu0kmS/us18jEmdv1a4Wi//BZ/DTVYEcH0XhLM5NYdpDHja3gt57VrZLcNAO2WGA+KpWsDBaHt6eA==
@@ -7397,21 +8791,38 @@
dependencies:
wrappy "1"
-onetime@^2.0.0:
- version "2.0.1"
- resolved "https://registry.yarnpkg.com/onetime/-/onetime-2.0.1.tgz#067428230fd67443b2794b22bba528b6867962d4"
- integrity sha1-BnQoIw/WdEOyeUsiu6UotoZ5YtQ=
+onetime@^5.1.0:
+ version "5.1.0"
+ resolved "https://registry.yarnpkg.com/onetime/-/onetime-5.1.0.tgz#fff0f3c91617fe62bb50189636e99ac8a6df7be5"
+ integrity sha512-5NcSkPHhwTVFIQN+TUqXoS5+dlElHXdpAWu9I0HP20YOtIi+aZ0Ct82jdlILDxjLEAWwvm+qj1m6aEtsDVmm6Q==
dependencies:
- mimic-fn "^1.0.0"
+ mimic-fn "^2.1.0"
-open@^6.3.0:
+open-editor@^2.0.1:
+ version "2.0.1"
+ resolved "https://registry.yarnpkg.com/open-editor/-/open-editor-2.0.1.tgz#d001055770fbf6f6ee73c18f224915f444be863c"
+ integrity sha512-B3KdD7Pl8jYdpBSBBbdYaqVUI3whQjLl1G1+CvhNc8+d7GzKRUq+VuCIx1thxGiqD2oBGRvsZz7QWrBsFP2yVA==
+ dependencies:
+ env-editor "^0.4.0"
+ line-column-path "^2.0.0"
+ open "^6.2.0"
+
+open@^6.2.0:
version "6.4.0"
resolved "https://registry.yarnpkg.com/open/-/open-6.4.0.tgz#5c13e96d0dc894686164f18965ecfe889ecfc8a9"
integrity sha512-IFenVPgF70fSm1keSd2iDBIDIBZkroLeuffXq+wKTzTJlBpesFWojV9lb8mzOfaAzM1sr7HQHuO0vtV0zYekGg==
dependencies:
is-wsl "^1.1.0"
-opn@^5.1.0:
+open@^7.0.2:
+ version "7.0.3"
+ resolved "https://registry.yarnpkg.com/open/-/open-7.0.3.tgz#db551a1af9c7ab4c7af664139930826138531c48"
+ integrity sha512-sP2ru2v0P290WFfv49Ap8MF6PkzGNnGlAwHweB4WR4mr5d2d0woiCluUeJ218w7/+PmoBy9JmYgD5A4mLcWOFA==
+ dependencies:
+ is-docker "^2.0.0"
+ is-wsl "^2.1.1"
+
+opn@^5.5.0:
version "5.5.0"
resolved "https://registry.yarnpkg.com/opn/-/opn-5.5.0.tgz#fc7164fab56d235904c51c3b27da6758ca3b9bfc"
integrity sha512-PqHpggC9bLV0VeWcdKhkpxY+3JTzetLSqTCWL/z/tFIbI6G8JCjondXklT1JinczLz2Xib62sSp0T/gKT4KksA==
@@ -7426,15 +8837,15 @@
minimist "~0.0.1"
wordwrap "~0.0.2"
-optimize-css-assets-webpack-plugin@5.0.1:
- version "5.0.1"
- resolved "https://registry.yarnpkg.com/optimize-css-assets-webpack-plugin/-/optimize-css-assets-webpack-plugin-5.0.1.tgz#9eb500711d35165b45e7fd60ba2df40cb3eb9159"
- integrity sha512-Rqm6sSjWtx9FchdP0uzTQDc7GXDKnwVEGoSxjezPkzMewx7gEWE9IMUYKmigTRC4U3RaNSwYVnUDLuIdtTpm0A==
+optimize-css-assets-webpack-plugin@5.0.3:
+ version "5.0.3"
+ resolved "https://registry.yarnpkg.com/optimize-css-assets-webpack-plugin/-/optimize-css-assets-webpack-plugin-5.0.3.tgz#e2f1d4d94ad8c0af8967ebd7cf138dcb1ef14572"
+ integrity sha512-q9fbvCRS6EYtUKKSwI87qm2IxlyJK5b4dygW1rKUBT6mMDhdG5e5bZT63v6tnJR9F9FB/H5a0HTmtw+laUBxKA==
dependencies:
- cssnano "^4.1.0"
+ cssnano "^4.1.10"
last-call-webpack-plugin "^3.0.0"
-optionator@^0.8.1, optionator@^0.8.2:
+optionator@^0.8.1, optionator@^0.8.3:
version "0.8.3"
resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.8.3.tgz#84fa1d036fe9d3c7e21d99884b601167ec8fb495"
integrity sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==
@@ -7513,6 +8924,13 @@
dependencies:
p-try "^2.0.0"
+p-limit@^2.2.0, p-limit@^2.2.2:
+ version "2.3.0"
+ resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.3.0.tgz#3dd33c647a214fdfffd835933eb086da0dc21db1"
+ integrity sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==
+ dependencies:
+ p-try "^2.0.0"
+
p-locate@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-2.0.0.tgz#20a0103b222a70c8fd39cc2e580680f3dde5ec43"
@@ -7527,16 +8945,42 @@
dependencies:
p-limit "^2.0.0"
-p-map@^1.1.1:
- version "1.2.0"
- resolved "https://registry.yarnpkg.com/p-map/-/p-map-1.2.0.tgz#e4e94f311eabbc8633a1e79908165fca26241b6b"
- integrity sha512-r6zKACMNhjPJMTl8KcFH4li//gkrXWfbD6feV8l6doRHlzljFWGJ2AP6iKaCJXyZmAUMOPtvbW7EXkbWO/pLEA==
+p-locate@^4.1.0:
+ version "4.1.0"
+ resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-4.1.0.tgz#a3428bb7088b3a60292f66919278b7c297ad4f07"
+ integrity sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==
+ dependencies:
+ p-limit "^2.2.0"
+
+p-map@^2.0.0:
+ version "2.1.0"
+ resolved "https://registry.yarnpkg.com/p-map/-/p-map-2.1.0.tgz#310928feef9c9ecc65b68b17693018a665cea175"
+ integrity sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==
+
+p-map@^3.0.0:
+ version "3.0.0"
+ resolved "https://registry.yarnpkg.com/p-map/-/p-map-3.0.0.tgz#d704d9af8a2ba684e2600d9a215983d4141a979d"
+ integrity sha512-d3qXVTF/s+W+CdJ5A29wywV2n8CQQYahlgz2bFiA+4eVNJbHJodPZ+/gXwPGh0bOqA+j8S+6+ckmvLGPk1QpxQ==
+ dependencies:
+ aggregate-error "^3.0.0"
p-reduce@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/p-reduce/-/p-reduce-1.0.0.tgz#18c2b0dd936a4690a529f8231f58a0fdb6a47dfa"
integrity sha1-GMKw3ZNqRpClKfgjH1ig/bakffo=
+p-reduce@^2.1.0:
+ version "2.1.0"
+ resolved "https://registry.yarnpkg.com/p-reduce/-/p-reduce-2.1.0.tgz#09408da49507c6c274faa31f28df334bc712b64a"
+ integrity sha512-2USApvnsutq8uoxZBGbbWM0JIYLiEMJ9RlaN7fAzVNb9OZN0SHjjTTfIcb667XynS5Y1VhwDJVDa72TnPzAYWw==
+
+p-retry@^3.0.1:
+ version "3.0.1"
+ resolved "https://registry.yarnpkg.com/p-retry/-/p-retry-3.0.1.tgz#316b4c8893e2c8dc1cfa891f406c4b422bebf328"
+ integrity sha512-XE6G4+YTTkT2a0UWb2kjZe8xNwf8bIbnqpc/IS/idOBVhyves0mK5OJgeocjx7q5pvX/6m23xuzVPYT1uGM73w==
+ dependencies:
+ retry "^0.12.0"
+
p-try@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3"
@@ -7571,12 +9015,13 @@
inherits "^2.0.3"
readable-stream "^2.1.5"
-param-case@2.1.x:
- version "2.1.1"
- resolved "https://registry.yarnpkg.com/param-case/-/param-case-2.1.1.tgz#df94fd8cf6531ecf75e6bef9a0858fbc72be2247"
- integrity sha1-35T9jPZTHs915r75oIWPvHK+Ikc=
+param-case@^3.0.3:
+ version "3.0.3"
+ resolved "https://registry.yarnpkg.com/param-case/-/param-case-3.0.3.tgz#4be41f8399eff621c56eebb829a5e451d9801238"
+ integrity sha512-VWBVyimc1+QrzappRs7waeN2YmoZFCGXWASRYX1/rGHtXqEcrGEIDm+jqIwFa2fRXNgQEwrxaYuIrX0WcAguTA==
dependencies:
- no-case "^2.2.0"
+ dot-case "^3.0.3"
+ tslib "^1.10.0"
parent-module@^1.0.0:
version "1.0.1"
@@ -7612,6 +9057,16 @@
error-ex "^1.3.1"
json-parse-better-errors "^1.0.1"
+parse-json@^5.0.0:
+ version "5.0.0"
+ resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-5.0.0.tgz#73e5114c986d143efa3712d4ea24db9a4266f60f"
+ integrity sha512-OOY5b7PAEFV0E2Fir1KOkxchnZNCdowAJgQ5NuxjpBKTRP3pQhwkrkxqQjeoKJ+fO7bCpmIZaogI4eZGDMEGOw==
+ dependencies:
+ "@babel/code-frame" "^7.0.0"
+ error-ex "^1.3.1"
+ json-parse-better-errors "^1.0.1"
+ lines-and-columns "^1.1.6"
+
parse-ms@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/parse-ms/-/parse-ms-2.1.0.tgz#348565a753d4391fa524029956b172cb7753097d"
@@ -7632,6 +9087,14 @@
resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4"
integrity sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==
+pascal-case@^3.1.1:
+ version "3.1.1"
+ resolved "https://registry.yarnpkg.com/pascal-case/-/pascal-case-3.1.1.tgz#5ac1975133ed619281e88920973d2cd1f279de5f"
+ integrity sha512-XIeHKqIrsquVTQL2crjq3NfJUxmdLasn3TYOU0VBM+UX2a6ztAWBlJQBePLGY7VHW8+2dRadeIPK5+KImwTxQA==
+ dependencies:
+ no-case "^3.0.3"
+ tslib "^1.10.0"
+
pascalcase@^0.1.1:
version "0.1.1"
resolved "https://registry.yarnpkg.com/pascalcase/-/pascalcase-0.1.1.tgz#b363e55e8006ca6fe21784d2db22bd15d7917f14"
@@ -7659,6 +9122,11 @@
resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-3.0.0.tgz#ce0ebeaa5f78cb18925ea7d810d7b59b010fd515"
integrity sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=
+path-exists@^4.0.0:
+ version "4.0.0"
+ resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-4.0.0.tgz#513bdbe2d3b95d7762e8c1137efa195c6c61b5b3"
+ integrity sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==
+
path-is-absolute@^1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f"
@@ -7674,6 +9142,11 @@
resolved "https://registry.yarnpkg.com/path-key/-/path-key-2.0.1.tgz#411cadb574c5a140d3a4b1910d40d80cc9f40b40"
integrity sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=
+path-key@^3.1.0:
+ version "3.1.1"
+ resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375"
+ integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==
+
path-parse@^1.0.6:
version "1.0.6"
resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.6.tgz#d62dbb5679405d72c4737ec58600e9ddcf06d24c"
@@ -7705,6 +9178,11 @@
dependencies:
pify "^3.0.0"
+path-type@^4.0.0:
+ version "4.0.0"
+ resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b"
+ integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==
+
pbkdf2@^3.0.3:
version "3.0.17"
resolved "https://registry.yarnpkg.com/pbkdf2/-/pbkdf2-3.0.17.tgz#976c206530617b14ebb32114239f7b09336e93a6"
@@ -7721,6 +9199,11 @@
resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b"
integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=
+picomatch@^2.0.4, picomatch@^2.0.5, picomatch@^2.2.1:
+ version "2.2.2"
+ resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.2.2.tgz#21f333e9b6b8eaff02468f5146ea406d345f4dad"
+ integrity sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==
+
pidtree@^0.3.0:
version "0.3.0"
resolved "https://registry.yarnpkg.com/pidtree/-/pidtree-0.3.0.tgz#f6fada10fccc9f99bf50e90d0b23d72c9ebc2e6b"
@@ -7781,7 +9264,21 @@
dependencies:
find-up "^3.0.0"
-pkg-up@2.0.0:
+pkg-dir@^4.1.0, pkg-dir@^4.2.0:
+ version "4.2.0"
+ resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-4.2.0.tgz#f099133df7ede422e81d1d8448270eeb3e4261f3"
+ integrity sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==
+ dependencies:
+ find-up "^4.0.0"
+
+pkg-up@3.1.0, pkg-up@^3.1.0:
+ version "3.1.0"
+ resolved "https://registry.yarnpkg.com/pkg-up/-/pkg-up-3.1.0.tgz#100ec235cc150e4fd42519412596a28512a0def5"
+ integrity sha512-nDywThFk1i4BQK4twPQ6TA4RT8bDY96yeuCVBWL3ePARCiEKDRSrNGbFIgUJpLp+XeIR65v8ra7WuJOFUBtkMA==
+ dependencies:
+ find-up "^3.0.0"
+
+pkg-up@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/pkg-up/-/pkg-up-2.0.0.tgz#c819ac728059a461cab1c3889a2be3c49a004d7f"
integrity sha1-yBmscoBZpGHKscOImivjxJoATX8=
@@ -7795,6 +9292,13 @@
dependencies:
semver-compare "^1.0.0"
+plur@^3.0.1:
+ version "3.1.1"
+ resolved "https://registry.yarnpkg.com/plur/-/plur-3.1.1.tgz#60267967866a8d811504fe58f2faaba237546a5b"
+ integrity sha512-t1Ax8KUvV3FFII8ltczPn2tJdjqbd1sIzu6t4JL7nQ3EyeL/lTrj5PWKb06ic5/6XYDr65rQ4uzQEGN70/6X5w==
+ dependencies:
+ irregular-plurals "^2.0.0"
+
pluralize@^8.0.0:
version "8.0.0"
resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-8.0.0.tgz#1a6fa16a38d12a1901e0320fa017051c539ce3b1"
@@ -7805,17 +9309,17 @@
resolved "https://registry.yarnpkg.com/pn/-/pn-1.1.0.tgz#e2f4cef0e219f463c179ab37463e4e1ecdccbafb"
integrity sha512-2qHaIQr2VLRFoxe2nASzsV6ef4yOOH+Fi9FBOVH6cqeSgUnoyySPZkxzLuzd+RYOQTRpROA0ztTMqxROKSb/nA==
-pnp-webpack-plugin@1.2.1:
- version "1.2.1"
- resolved "https://registry.yarnpkg.com/pnp-webpack-plugin/-/pnp-webpack-plugin-1.2.1.tgz#cd9d698df2a6fcf7255093c1c9511adf65b9421b"
- integrity sha512-W6GctK7K2qQiVR+gYSv/Gyt6jwwIH4vwdviFqx+Y2jAtVf5eZyYIDf5Ac2NCDMBiX5yWscBLZElPTsyA1UtVVA==
+pnp-webpack-plugin@1.6.4:
+ version "1.6.4"
+ resolved "https://registry.yarnpkg.com/pnp-webpack-plugin/-/pnp-webpack-plugin-1.6.4.tgz#c9711ac4dc48a685dabafc86f8b6dd9f8df84149"
+ integrity sha512-7Wjy+9E3WwLOEL30D+m8TSTF7qJJUJLONBnwQp0518siuMxUQUbgZwssaFX+QKlZkjHZcw/IpZCt/H0srrntSg==
dependencies:
- ts-pnp "^1.0.0"
+ ts-pnp "^1.1.6"
-portfinder@^1.0.9:
- version "1.0.25"
- resolved "https://registry.yarnpkg.com/portfinder/-/portfinder-1.0.25.tgz#254fd337ffba869f4b9d37edc298059cb4d35eca"
- integrity sha512-6ElJnHBbxVA1XSLgBp7G1FiCkQdlqGzuF7DswL5tcea+E8UpuvPU7beVAjjRwCioTS9ZluNbu+ZyRvgTsmqEBg==
+portfinder@^1.0.25:
+ version "1.0.26"
+ resolved "https://registry.yarnpkg.com/portfinder/-/portfinder-1.0.26.tgz#475658d56ca30bed72ac7f1378ed350bd1b64e70"
+ integrity sha512-Xi7mKxJHHMI3rIUrnm/jjUgwhbYMkp/XKEcZX3aG4BrumLpq3nmoQMX+ClYnDZnZ/New7IatC1no5RX0zo1vXQ==
dependencies:
async "^2.6.2"
debug "^3.1.1"
@@ -7834,12 +9338,12 @@
postcss "^7.0.2"
postcss-selector-parser "^5.0.0"
-postcss-browser-comments@^2.0.0:
- version "2.0.0"
- resolved "https://registry.yarnpkg.com/postcss-browser-comments/-/postcss-browser-comments-2.0.0.tgz#dc48d6a8ddbff188a80a000b7393436cb18aed88"
- integrity sha512-xGG0UvoxwBc4Yx4JX3gc0RuDl1kc4bVihCzzk6UC72YPfq5fu3c717Nu8Un3nvnq1BJ31gBnFXIG/OaUTnpHgA==
+postcss-browser-comments@^3.0.0:
+ version "3.0.0"
+ resolved "https://registry.yarnpkg.com/postcss-browser-comments/-/postcss-browser-comments-3.0.0.tgz#1248d2d935fb72053c8e1f61a84a57292d9f65e9"
+ integrity sha512-qfVjLfq7HFd2e0HW4s1dvU8X080OZdG46fFbIBFjW7US7YPDcWfRvdElvwMJr2LI6hMmD+7LnH2HcmXTs+uOig==
dependencies:
- postcss "^7.0.2"
+ postcss "^7"
postcss-calc@^7.0.1:
version "7.0.1"
@@ -7868,7 +9372,7 @@
postcss "^7.0.5"
postcss-values-parser "^2.0.0"
-postcss-color-hex-alpha@^5.0.2:
+postcss-color-hex-alpha@^5.0.3:
version "5.0.3"
resolved "https://registry.yarnpkg.com/postcss-color-hex-alpha/-/postcss-color-hex-alpha-5.0.3.tgz#a8d9ca4c39d497c9661e374b9c51899ef0f87388"
integrity sha512-PF4GDel8q3kkreVXKLAGNpHKilXsZ6xuu+mOQMHWHLPNyjiUBOr75sp5ZKJfmv1MCus5/DWUGcK9hm6qHEnXYw==
@@ -7912,14 +9416,14 @@
postcss "^7.0.0"
postcss-value-parser "^3.0.0"
-postcss-custom-media@^7.0.7:
+postcss-custom-media@^7.0.8:
version "7.0.8"
resolved "https://registry.yarnpkg.com/postcss-custom-media/-/postcss-custom-media-7.0.8.tgz#fffd13ffeffad73621be5f387076a28b00294e0c"
integrity sha512-c9s5iX0Ge15o00HKbuRuTqNndsJUbaXdiNsksnVH8H4gdc+zbLzr/UasOwNG6CTDpLFekVY4672eWdiiWu2GUg==
dependencies:
postcss "^7.0.14"
-postcss-custom-properties@^8.0.9:
+postcss-custom-properties@^8.0.11:
version "8.0.11"
resolved "https://registry.yarnpkg.com/postcss-custom-properties/-/postcss-custom-properties-8.0.11.tgz#2d61772d6e92f22f5e0d52602df8fae46fa30d97"
integrity sha512-nm+o0eLdYqdnJ5abAJeXp4CEU1c1k+eB2yMCvhgzsds/e0umabFrN6HoTy/8Q4K5ilxERdl/JD1LO5ANoYBeMA==
@@ -8148,29 +9652,30 @@
dependencies:
postcss "^7.0.5"
-postcss-modules-local-by-default@^2.0.6:
- version "2.0.6"
- resolved "https://registry.yarnpkg.com/postcss-modules-local-by-default/-/postcss-modules-local-by-default-2.0.6.tgz#dd9953f6dd476b5fd1ef2d8830c8929760b56e63"
- integrity sha512-oLUV5YNkeIBa0yQl7EYnxMgy4N6noxmiwZStaEJUSe2xPMcdNc8WmBQuQCx18H5psYbVxz8zoHk0RAAYZXP9gA==
+postcss-modules-local-by-default@^3.0.2:
+ version "3.0.2"
+ resolved "https://registry.yarnpkg.com/postcss-modules-local-by-default/-/postcss-modules-local-by-default-3.0.2.tgz#e8a6561be914aaf3c052876377524ca90dbb7915"
+ integrity sha512-jM/V8eqM4oJ/22j0gx4jrp63GSvDH6v86OqyTHHUvk4/k1vceipZsaymiZ5PvocqZOl5SFHiFJqjs3la0wnfIQ==
dependencies:
- postcss "^7.0.6"
- postcss-selector-parser "^6.0.0"
- postcss-value-parser "^3.3.1"
+ icss-utils "^4.1.1"
+ postcss "^7.0.16"
+ postcss-selector-parser "^6.0.2"
+ postcss-value-parser "^4.0.0"
-postcss-modules-scope@^2.1.0:
- version "2.1.1"
- resolved "https://registry.yarnpkg.com/postcss-modules-scope/-/postcss-modules-scope-2.1.1.tgz#33d4fc946602eb5e9355c4165d68a10727689dba"
- integrity sha512-OXRUPecnHCg8b9xWvldG/jUpRIGPNRka0r4D4j0ESUU2/5IOnpsjfPPmDprM3Ih8CgZ8FXjWqaniK5v4rWt3oQ==
+postcss-modules-scope@^2.1.1:
+ version "2.2.0"
+ resolved "https://registry.yarnpkg.com/postcss-modules-scope/-/postcss-modules-scope-2.2.0.tgz#385cae013cc7743f5a7d7602d1073a89eaae62ee"
+ integrity sha512-YyEgsTMRpNd+HmyC7H/mh3y+MeFWevy7V1evVhJWewmMbjDHIbZbOXICC2y+m1xI1UVfIT1HMW/O04Hxyu9oXQ==
dependencies:
postcss "^7.0.6"
postcss-selector-parser "^6.0.0"
-postcss-modules-values@^2.0.0:
- version "2.0.0"
- resolved "https://registry.yarnpkg.com/postcss-modules-values/-/postcss-modules-values-2.0.0.tgz#479b46dc0c5ca3dc7fa5270851836b9ec7152f64"
- integrity sha512-Ki7JZa7ff1N3EIMlPnGTZfUMe69FFwiQPnVSXC9mnn3jozCRBYIxiZd44yJOV2AmabOo4qFf8s0dC/+lweG7+w==
+postcss-modules-values@^3.0.0:
+ version "3.0.0"
+ resolved "https://registry.yarnpkg.com/postcss-modules-values/-/postcss-modules-values-3.0.0.tgz#5b5000d6ebae29b4255301b4a3a54574423e7f10"
+ integrity sha512-1//E5jCBrZ9DmRX+zCtmQtRSV6PV42Ix7Bzj9GbwJceduuf7IqP8MgeTXuRDHOWj2m0VzZD5+roFWDuU8RQjcg==
dependencies:
- icss-replace-symbols "^1.1.0"
+ icss-utils "^4.0.0"
postcss "^7.0.6"
postcss-nesting@^7.0.0:
@@ -8261,15 +9766,16 @@
postcss "^7.0.0"
postcss-value-parser "^3.0.0"
-postcss-normalize@7.0.1:
- version "7.0.1"
- resolved "https://registry.yarnpkg.com/postcss-normalize/-/postcss-normalize-7.0.1.tgz#eb51568d962b8aa61a8318383c8bb7e54332282e"
- integrity sha512-NOp1fwrG+6kVXWo7P9SizCHX6QvioxFD/hZcI2MLxPmVnFJFC0j0DDpIuNw2tUDeCFMni59gCVgeJ1/hYhj2OQ==
+postcss-normalize@8.0.1:
+ version "8.0.1"
+ resolved "https://registry.yarnpkg.com/postcss-normalize/-/postcss-normalize-8.0.1.tgz#90e80a7763d7fdf2da6f2f0f82be832ce4f66776"
+ integrity sha512-rt9JMS/m9FHIRroDDBGSMsyW1c0fkvOJPy62ggxSHUldJO7B195TqFMqIf+lY5ezpDcYOV4j86aUp3/XbxzCCQ==
dependencies:
- "@csstools/normalize.css" "^9.0.1"
- browserslist "^4.1.1"
- postcss "^7.0.2"
- postcss-browser-comments "^2.0.0"
+ "@csstools/normalize.css" "^10.1.0"
+ browserslist "^4.6.2"
+ postcss "^7.0.17"
+ postcss-browser-comments "^3.0.0"
+ sanitize.css "^10.0.0"
postcss-ordered-values@^4.1.2:
version "4.1.2"
@@ -8302,27 +9808,27 @@
postcss "^7.0.2"
postcss-values-parser "^2.0.0"
-postcss-preset-env@6.6.0:
- version "6.6.0"
- resolved "https://registry.yarnpkg.com/postcss-preset-env/-/postcss-preset-env-6.6.0.tgz#642e7d962e2bdc2e355db117c1eb63952690ed5b"
- integrity sha512-I3zAiycfqXpPIFD6HXhLfWXIewAWO8emOKz+QSsxaUZb9Dp8HbF5kUf+4Wy/AxR33o+LRoO8blEWCHth0ZsCLA==
+postcss-preset-env@6.7.0:
+ version "6.7.0"
+ resolved "https://registry.yarnpkg.com/postcss-preset-env/-/postcss-preset-env-6.7.0.tgz#c34ddacf8f902383b35ad1e030f178f4cdf118a5"
+ integrity sha512-eU4/K5xzSFwUFJ8hTdTQzo2RBLbDVt83QZrAvI07TULOkmyQlnYlpwep+2yIK+K+0KlZO4BvFcleOCCcUtwchg==
dependencies:
- autoprefixer "^9.4.9"
- browserslist "^4.4.2"
- caniuse-lite "^1.0.30000939"
+ autoprefixer "^9.6.1"
+ browserslist "^4.6.4"
+ caniuse-lite "^1.0.30000981"
css-blank-pseudo "^0.1.4"
css-has-pseudo "^0.10.0"
css-prefers-color-scheme "^3.1.1"
- cssdb "^4.3.0"
- postcss "^7.0.14"
+ cssdb "^4.4.0"
+ postcss "^7.0.17"
postcss-attribute-case-insensitive "^4.0.1"
postcss-color-functional-notation "^2.0.1"
postcss-color-gray "^5.0.0"
- postcss-color-hex-alpha "^5.0.2"
+ postcss-color-hex-alpha "^5.0.3"
postcss-color-mod-function "^3.0.3"
postcss-color-rebeccapurple "^4.0.1"
- postcss-custom-media "^7.0.7"
- postcss-custom-properties "^8.0.9"
+ postcss-custom-media "^7.0.8"
+ postcss-custom-properties "^8.0.11"
postcss-custom-selectors "^5.1.2"
postcss-dir-pseudo-class "^5.0.0"
postcss-double-position-gradients "^1.0.0"
@@ -8421,7 +9927,7 @@
indexes-of "^1.0.1"
uniq "^1.0.1"
-postcss-selector-parser@^6.0.0:
+postcss-selector-parser@^6.0.0, postcss-selector-parser@^6.0.2:
version "6.0.2"
resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-6.0.2.tgz#934cf799d016c83411859e09dcecade01286ec5c"
integrity sha512-36P2QR59jDTOAiIkqEprfJDsoNrvwFei3eCqKd1Y0tUsBimsq39BLp7RD+JWny3WgB1zGhJX8XVePwm9k4wdBg==
@@ -8449,11 +9955,16 @@
postcss "^7.0.0"
uniqs "^2.0.0"
-postcss-value-parser@^3.0.0, postcss-value-parser@^3.3.0, postcss-value-parser@^3.3.1:
+postcss-value-parser@^3.0.0, postcss-value-parser@^3.3.1:
version "3.3.1"
resolved "https://registry.yarnpkg.com/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz#9ff822547e2893213cf1c30efa51ac5fd1ba8281"
integrity sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==
+postcss-value-parser@^4.0.0, postcss-value-parser@^4.0.3:
+ version "4.1.0"
+ resolved "https://registry.yarnpkg.com/postcss-value-parser/-/postcss-value-parser-4.1.0.tgz#443f6a20ced6481a2bda4fa8532a6e55d789a2cb"
+ integrity sha512-97DXOFbQJhk71ne5/Mt6cOu6yxsSfM0QGQyl0L25Gca4yGWEGJaig7l7gbCX623VqTBNGLRLaVUCnNkcedlRSQ==
+
postcss-value-parser@^4.0.2:
version "4.0.2"
resolved "https://registry.yarnpkg.com/postcss-value-parser/-/postcss-value-parser-4.0.2.tgz#482282c09a42706d1fc9a069b73f44ec08391dc9"
@@ -8468,6 +9979,24 @@
indexes-of "^1.0.1"
uniq "^1.0.1"
+postcss@7.0.21:
+ version "7.0.21"
+ resolved "https://registry.yarnpkg.com/postcss/-/postcss-7.0.21.tgz#06bb07824c19c2021c5d056d5b10c35b989f7e17"
+ integrity sha512-uIFtJElxJo29QC753JzhidoAhvp/e/Exezkdhfmt8AymWT6/5B7W1WmponYWkHk2eg6sONyTch0A3nkMPun3SQ==
+ dependencies:
+ chalk "^2.4.2"
+ source-map "^0.6.1"
+ supports-color "^6.1.0"
+
+postcss@^7, postcss@^7.0.16, postcss@^7.0.27:
+ version "7.0.29"
+ resolved "https://registry.yarnpkg.com/postcss/-/postcss-7.0.29.tgz#d3a903872bd52280b83bce38cdc83ce55c06129e"
+ integrity sha512-ba0ApvR3LxGvRMMiUa9n0WR4HjzcYm7tS+ht4/2Nd0NLtHpPIH77fuB9Xh1/yJVz9O/E/95Y/dn8ygWsyffXtw==
+ dependencies:
+ chalk "^2.4.2"
+ source-map "^0.6.1"
+ supports-color "^6.1.0"
+
postcss@^7.0.0, postcss@^7.0.1, postcss@^7.0.14, postcss@^7.0.17, postcss@^7.0.2, postcss@^7.0.23, postcss@^7.0.5, postcss@^7.0.6:
version "7.0.26"
resolved "https://registry.yarnpkg.com/postcss/-/postcss-7.0.26.tgz#5ed615cfcab35ba9bbb82414a4fa88ea10429587"
@@ -8482,11 +10011,28 @@
resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54"
integrity sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=
+prepend-http@^1.0.0:
+ version "1.0.4"
+ resolved "https://registry.yarnpkg.com/prepend-http/-/prepend-http-1.0.4.tgz#d4f4562b0ce3696e41ac52d0e002e57a635dc6dc"
+ integrity sha1-1PRWKwzjaW5BrFLQ4ALlemNdxtw=
+
prepend-http@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/prepend-http/-/prepend-http-2.0.0.tgz#e92434bfa5ea8c19f41cdfd401d741a3c819d897"
integrity sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc=
+prettier-linter-helpers@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/prettier-linter-helpers/-/prettier-linter-helpers-1.0.0.tgz#d23d41fe1375646de2d0104d3454a3008802cf7b"
+ integrity sha512-GbK2cP9nraSSUF9N2XwUwqfzlAFlMNYYl+ShE/V+H8a9uNl/oUqB1w2EL54Jh0OlyRSd8RfWYJ3coVS4TROP2w==
+ dependencies:
+ fast-diff "^1.1.2"
+
+prettier@2.0.4:
+ version "2.0.4"
+ resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.0.4.tgz#2d1bae173e355996ee355ec9830a7a1ee05457ef"
+ integrity sha512-SVJIQ51spzFDvh4fIbCLvciiDMCrRhlN3mbZvv/+ycjvmF5E73bKdGfU8QDLNmjYJf+lsGnDBC4UUnvTe5OO0w==
+
pretty-bytes@^5.1.0, pretty-bytes@^5.3.0:
version "5.3.0"
resolved "https://registry.yarnpkg.com/pretty-bytes/-/pretty-bytes-5.3.0.tgz#f2849e27db79fb4d6cfe24764fc4134f165989f2"
@@ -8517,7 +10063,7 @@
dependencies:
parse-ms "^2.1.0"
-private@^0.1.6:
+private@^0.1.8:
version "0.1.8"
resolved "https://registry.yarnpkg.com/private/-/private-0.1.8.tgz#2381edb3689f7a53d653190060fcf822d2f368ff"
integrity sha512-VvivMrbvd2nKkiG38qjULzlc+4Vx4wm/whI9pQD35YrARNnhxeiRktSOhSukRLFNlzg6Br/cJPet5J/u19r/mg==
@@ -8573,6 +10119,11 @@
object-assign "^4.1.1"
react-is "^16.8.1"
+proto-props@^2.0.0:
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/proto-props/-/proto-props-2.0.0.tgz#8ac6e6dec658545815c623a3bc81580deda9a181"
+ integrity sha512-2yma2tog9VaRZY2mn3Wq51uiSW4NcPYT1cQdBagwyrznrilKSZwIZ0UG3ZPL/mx+axEns0hE35T5ufOYZXEnBQ==
+
proxy-addr@~2.0.5:
version "2.0.5"
resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.5.tgz#34cbd64a2d81f4b1fd21e76f9f06c8a45299ee34"
@@ -8648,6 +10199,13 @@
resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec"
integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==
+pupa@^2.0.1:
+ version "2.0.1"
+ resolved "https://registry.yarnpkg.com/pupa/-/pupa-2.0.1.tgz#dbdc9ff48ffbea4a26a069b6f9f7abb051008726"
+ integrity sha512-hEJH0s8PXLY/cdXh66tNEQGndDrIKNqNC5xmrysZy3i5C3oEoLna7YAOad+7u125+zH1HNXUmGEkrhb3c2VriA==
+ dependencies:
+ escape-goat "^2.0.0"
+
q@^1.1.2:
version "1.5.1"
resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7"
@@ -8663,6 +10221,14 @@
resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36"
integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==
+query-string@^4.1.0:
+ version "4.3.4"
+ resolved "https://registry.yarnpkg.com/query-string/-/query-string-4.3.4.tgz#bbb693b9ca915c232515b228b1a02b609043dbeb"
+ integrity sha1-u7aTucqRXCMlFbIosaArYJBD2+s=
+ dependencies:
+ object-assign "^4.1.0"
+ strict-uri-encode "^1.0.0"
+
querystring-es3@^0.2.0:
version "0.2.1"
resolved "https://registry.yarnpkg.com/querystring-es3/-/querystring-es3-0.2.1.tgz#9ec61f79049875707d69414596fd907a4d711e73"
@@ -8678,6 +10244,11 @@
resolved "https://registry.yarnpkg.com/querystringify/-/querystringify-2.1.1.tgz#60e5a5fd64a7f8bfa4d2ab2ed6fdf4c85bad154e"
integrity sha512-w7fLxIRCRT7U8Qu53jQnJyPkYZIaR4n5151KMfcJlO/A9397Wxb1amJvROTK6TOnp7PfoAmg/qXiNHI+08jRfA==
+quick-lru@^1.0.0:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-1.1.0.tgz#4360b17c61136ad38078397ff11416e186dcfbb8"
+ integrity sha1-Q2CxfGETatOAeDl/8RQW4Ybc+7g=
+
raf@^3.4.0, raf@^3.4.1:
version "3.4.1"
resolved "https://registry.yarnpkg.com/raf/-/raf-3.4.1.tgz#0742e99a4a6552f445d73e3ee0328af0ff1ede39"
@@ -9157,12 +10728,12 @@
minimist "^1.2.0"
strip-json-comments "~2.0.1"
-react-app-polyfill@^1.0.0:
- version "1.0.5"
- resolved "https://registry.yarnpkg.com/react-app-polyfill/-/react-app-polyfill-1.0.5.tgz#59c7377a0b9ed25692eeaca7ad9b12ef2d064709"
- integrity sha512-RcbV6+msbvZJZUIK/LX3UafPtoaDSJgUWu4sqBxHKTVmBsnlU2QWCKJRBRmgjxu+ivW/GPINbPWRM4Ppa6Lbgw==
+react-app-polyfill@^1.0.6:
+ version "1.0.6"
+ resolved "https://registry.yarnpkg.com/react-app-polyfill/-/react-app-polyfill-1.0.6.tgz#890f8d7f2842ce6073f030b117de9130a5f385f0"
+ integrity sha512-OfBnObtnGgLGfweORmdZbyEz+3dgVePQBb3zipiaDsMHV1NpWm0rDFYIVXFV/AK+x4VIIfWHhrdMIeoTLyRr2g==
dependencies:
- core-js "^3.4.1"
+ core-js "^3.5.0"
object-assign "^4.1.1"
promise "^8.0.3"
raf "^3.4.1"
@@ -9176,35 +10747,34 @@
dependencies:
semver "^5.6.0"
-react-dev-utils@^9.0.0:
- version "9.1.0"
- resolved "https://registry.yarnpkg.com/react-dev-utils/-/react-dev-utils-9.1.0.tgz#3ad2bb8848a32319d760d0a84c56c14bdaae5e81"
- integrity sha512-X2KYF/lIGyGwP/F/oXgGDF24nxDA2KC4b7AFto+eqzc/t838gpSGiaU8trTqHXOohuLxxc5qi1eDzsl9ucPDpg==
+react-dev-utils@^10.2.1:
+ version "10.2.1"
+ resolved "https://registry.yarnpkg.com/react-dev-utils/-/react-dev-utils-10.2.1.tgz#f6de325ae25fa4d546d09df4bb1befdc6dd19c19"
+ integrity sha512-XxTbgJnYZmxuPtY3y/UV0D8/65NKkmaia4rXzViknVnZeVlklSh8u6TnaEYPfAi/Gh1TP4mEOXHI6jQOPbeakQ==
dependencies:
- "@babel/code-frame" "7.5.5"
+ "@babel/code-frame" "7.8.3"
address "1.1.2"
- browserslist "4.7.0"
+ browserslist "4.10.0"
chalk "2.4.2"
- cross-spawn "6.0.5"
+ cross-spawn "7.0.1"
detect-port-alt "1.1.6"
- escape-string-regexp "1.0.5"
- filesize "3.6.1"
- find-up "3.0.0"
- fork-ts-checker-webpack-plugin "1.5.0"
+ escape-string-regexp "2.0.0"
+ filesize "6.0.1"
+ find-up "4.1.0"
+ fork-ts-checker-webpack-plugin "3.1.1"
global-modules "2.0.0"
globby "8.0.2"
gzip-size "5.1.1"
immer "1.10.0"
- inquirer "6.5.0"
+ inquirer "7.0.4"
is-root "2.1.0"
loader-utils "1.2.3"
- open "^6.3.0"
- pkg-up "2.0.0"
- react-error-overlay "^6.0.3"
+ open "^7.0.2"
+ pkg-up "3.1.0"
+ react-error-overlay "^6.0.7"
recursive-readdir "2.2.2"
shell-quote "1.7.2"
- sockjs-client "1.4.0"
- strip-ansi "5.2.0"
+ strip-ansi "6.0.0"
text-table "0.2.0"
react-dom@^16.8.6:
@@ -9217,10 +10787,10 @@
prop-types "^15.6.2"
scheduler "^0.18.0"
-react-error-overlay@^6.0.3:
- version "6.0.4"
- resolved "https://registry.yarnpkg.com/react-error-overlay/-/react-error-overlay-6.0.4.tgz#0d165d6d27488e660bc08e57bdabaad741366f7a"
- integrity sha512-ueZzLmHltszTshDMwyfELDq8zOA803wQ1ZuzCccXa1m57k1PxSHfflPD5W9YIiTXLs0JTLzoj6o1LuM5N6zzNA==
+react-error-overlay@^6.0.7:
+ version "6.0.7"
+ resolved "https://registry.yarnpkg.com/react-error-overlay/-/react-error-overlay-6.0.7.tgz#1dcfb459ab671d53f660a991513cb2f0a0553108"
+ integrity sha512-TAv1KJFh3RhqxNvhzxj6LeT5NWklP6rDr2a0jaTfsZ5wSZWHOGeqQyejUp3xxLfPt2UpyJEcVQB/zyPcmonNFA==
react-is@^16.6.0, react-is@^16.7.0, react-is@^16.8.1, react-is@^16.8.4:
version "16.12.0"
@@ -9255,7 +10825,7 @@
tiny-invariant "^1.0.2"
tiny-warning "^1.0.0"
-react-router@5.1.2:
+react-router@5.1.2, react-router@^5.1.2:
version "5.1.2"
resolved "https://registry.yarnpkg.com/react-router/-/react-router-5.1.2.tgz#6ea51d789cb36a6be1ba5f7c0d48dd9e817d3418"
integrity sha512-yjEuMFy1ONK246B+rsa0cUam5OeAQ8pyclRDgpxuSCrAlJ1qN9uZ5IgyKC7gQg0w8OM50NXHEegPh/ks9YuR2A==
@@ -9271,63 +10841,65 @@
tiny-invariant "^1.0.2"
tiny-warning "^1.0.0"
-react-scripts@3.0.0:
- version "3.0.0"
- resolved "https://registry.yarnpkg.com/react-scripts/-/react-scripts-3.0.0.tgz#a715613ef3eace025907b409cec8505096e0233e"
- integrity sha512-F4HegoBuUKZvEzXYksQu05Y6vJqallhHkQUEL6M7OQ5rYLBQC/4MTK6km9ZZvEK9TqMy1XA8SSEJGJgTEr6bSQ==
+react-scripts@^3.1.2:
+ version "3.4.1"
+ resolved "https://registry.yarnpkg.com/react-scripts/-/react-scripts-3.4.1.tgz#f551298b5c71985cc491b9acf3c8e8c0ae3ada0a"
+ integrity sha512-JpTdi/0Sfd31mZA6Ukx+lq5j1JoKItX7qqEK4OiACjVQletM1P38g49d9/D0yTxp9FrSF+xpJFStkGgKEIRjlQ==
dependencies:
- "@babel/core" "7.4.3"
- "@svgr/webpack" "4.1.0"
- "@typescript-eslint/eslint-plugin" "1.6.0"
- "@typescript-eslint/parser" "1.6.0"
- babel-eslint "10.0.1"
- babel-jest "24.7.1"
- babel-loader "8.0.5"
- babel-plugin-named-asset-import "^0.3.2"
- babel-preset-react-app "^8.0.0"
- case-sensitive-paths-webpack-plugin "2.2.0"
- css-loader "2.1.1"
- dotenv "6.2.0"
- dotenv-expand "4.2.0"
- eslint "^5.16.0"
- eslint-config-react-app "^4.0.0"
- eslint-loader "2.1.2"
- eslint-plugin-flowtype "2.50.1"
- eslint-plugin-import "2.16.0"
- eslint-plugin-jsx-a11y "6.2.1"
- eslint-plugin-react "7.12.4"
- eslint-plugin-react-hooks "^1.5.0"
- file-loader "3.0.1"
- fs-extra "7.0.1"
- html-webpack-plugin "4.0.0-beta.5"
+ "@babel/core" "7.9.0"
+ "@svgr/webpack" "4.3.3"
+ "@typescript-eslint/eslint-plugin" "^2.10.0"
+ "@typescript-eslint/parser" "^2.10.0"
+ babel-eslint "10.1.0"
+ babel-jest "^24.9.0"
+ babel-loader "8.1.0"
+ babel-plugin-named-asset-import "^0.3.6"
+ babel-preset-react-app "^9.1.2"
+ camelcase "^5.3.1"
+ case-sensitive-paths-webpack-plugin "2.3.0"
+ css-loader "3.4.2"
+ dotenv "8.2.0"
+ dotenv-expand "5.1.0"
+ eslint "^6.6.0"
+ eslint-config-react-app "^5.2.1"
+ eslint-loader "3.0.3"
+ eslint-plugin-flowtype "4.6.0"
+ eslint-plugin-import "2.20.1"
+ eslint-plugin-jsx-a11y "6.2.3"
+ eslint-plugin-react "7.19.0"
+ eslint-plugin-react-hooks "^1.6.1"
+ file-loader "4.3.0"
+ fs-extra "^8.1.0"
+ html-webpack-plugin "4.0.0-beta.11"
identity-obj-proxy "3.0.0"
- is-wsl "^1.1.0"
- jest "24.7.1"
- jest-environment-jsdom-fourteen "0.1.0"
- jest-resolve "24.7.1"
- jest-watch-typeahead "0.3.0"
- mini-css-extract-plugin "0.5.0"
- optimize-css-assets-webpack-plugin "5.0.1"
- pnp-webpack-plugin "1.2.1"
+ jest "24.9.0"
+ jest-environment-jsdom-fourteen "1.0.1"
+ jest-resolve "24.9.0"
+ jest-watch-typeahead "0.4.2"
+ mini-css-extract-plugin "0.9.0"
+ optimize-css-assets-webpack-plugin "5.0.3"
+ pnp-webpack-plugin "1.6.4"
postcss-flexbugs-fixes "4.1.0"
postcss-loader "3.0.0"
- postcss-normalize "7.0.1"
- postcss-preset-env "6.6.0"
+ postcss-normalize "8.0.1"
+ postcss-preset-env "6.7.0"
postcss-safe-parser "4.0.1"
- react-app-polyfill "^1.0.0"
- react-dev-utils "^9.0.0"
- resolve "1.10.0"
- sass-loader "7.1.0"
- semver "6.0.0"
+ react-app-polyfill "^1.0.6"
+ react-dev-utils "^10.2.1"
+ resolve "1.15.0"
+ resolve-url-loader "3.1.1"
+ sass-loader "8.0.2"
+ semver "6.3.0"
style-loader "0.23.1"
- terser-webpack-plugin "1.2.3"
- url-loader "1.1.2"
- webpack "4.29.6"
- webpack-dev-server "3.2.1"
- webpack-manifest-plugin "2.0.4"
- workbox-webpack-plugin "4.2.0"
+ terser-webpack-plugin "2.3.5"
+ ts-pnp "1.1.6"
+ url-loader "2.3.0"
+ webpack "4.42.0"
+ webpack-dev-server "3.10.3"
+ webpack-manifest-plugin "2.2.0"
+ workbox-webpack-plugin "4.3.1"
optionalDependencies:
- fsevents "2.0.6"
+ fsevents "2.1.2"
react-slick@~0.25.2:
version "0.25.2"
@@ -9357,6 +10929,14 @@
find-up "^2.0.0"
read-pkg "^2.0.0"
+read-pkg-up@^3.0.0:
+ version "3.0.0"
+ resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-3.0.0.tgz#3ed496685dba0f8fe118d0691dc51f4a1ff96f07"
+ integrity sha1-PtSWaF26D4/hGNBpHcUfSh/5bwc=
+ dependencies:
+ find-up "^2.0.0"
+ read-pkg "^3.0.0"
+
read-pkg-up@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-4.0.0.tgz#1b221c6088ba7799601c808f91161c66e58f8978"
@@ -9365,6 +10945,15 @@
find-up "^3.0.0"
read-pkg "^3.0.0"
+read-pkg-up@^7.0.1:
+ version "7.0.1"
+ resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-7.0.1.tgz#f3a6135758459733ae2b95638056e1854e7ef507"
+ integrity sha512-zK0TB7Xd6JpCLmlLmufqykGE+/TlOePD6qKClNW7hHDKFh/J7/7gCWGR7joEQEW1bKq3a3yUZSObOoWLFQ4ohg==
+ dependencies:
+ find-up "^4.1.0"
+ read-pkg "^5.2.0"
+ type-fest "^0.8.1"
+
read-pkg@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/read-pkg/-/read-pkg-2.0.0.tgz#8ef1c0623c6a6db0dc6713c4bfac46332b2368f8"
@@ -9383,6 +10972,16 @@
normalize-package-data "^2.3.2"
path-type "^3.0.0"
+read-pkg@^5.2.0:
+ version "5.2.0"
+ resolved "https://registry.yarnpkg.com/read-pkg/-/read-pkg-5.2.0.tgz#7bf295438ca5a33e56cd30e053b34ee7250c93cc"
+ integrity sha512-Ug69mNOpfvKDAc2Q8DRpMjjzdtrnv9HcSMX+4VsZxD1aZ6ZzrIE7rlzXBtWTyhULSMKg076AW6WR5iZpD0JiOg==
+ dependencies:
+ "@types/normalize-package-data" "^2.4.0"
+ normalize-package-data "^2.5.0"
+ parse-json "^5.0.0"
+ type-fest "^0.6.0"
+
"readable-stream@1 || 2", readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.1.5, readable-stream@^2.2.2, readable-stream@^2.3.3, readable-stream@^2.3.6, readable-stream@~2.3.6:
version "2.3.7"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57"
@@ -9414,6 +11013,13 @@
micromatch "^3.1.10"
readable-stream "^2.0.2"
+readdirp@~3.4.0:
+ version "3.4.0"
+ resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.4.0.tgz#9fdccdf9e9155805449221ac645e8303ab5b9ada"
+ integrity sha512-0xe001vZBnJEK+uKcj8qOhyAKPzIT+gStxWr3LCB0DwcXR5NZJ3IaC+yGnHCYzB/S7ov3m3EEbZI2zeNvX+hGQ==
+ dependencies:
+ picomatch "^2.2.1"
+
realpath-native@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/realpath-native/-/realpath-native-1.1.0.tgz#2003294fea23fb0672f2476ebe22fcf498a2d65c"
@@ -9428,10 +11034,18 @@
dependencies:
minimatch "3.0.4"
-regenerate-unicode-properties@^8.1.0:
- version "8.1.0"
- resolved "https://registry.yarnpkg.com/regenerate-unicode-properties/-/regenerate-unicode-properties-8.1.0.tgz#ef51e0f0ea4ad424b77bf7cb41f3e015c70a3f0e"
- integrity sha512-LGZzkgtLY79GeXLm8Dp0BVLdQlWICzBnJz/ipWUgo59qBaZ+BHtq51P2q1uVZlppMuUAT37SDk39qUbjTWB7bA==
+redent@^2.0.0:
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/redent/-/redent-2.0.0.tgz#c1b2007b42d57eb1389079b3c8333639d5e1ccaa"
+ integrity sha1-wbIAe0LVfrE4kHmzyDM2OdXhzKo=
+ dependencies:
+ indent-string "^3.0.0"
+ strip-indent "^2.0.0"
+
+regenerate-unicode-properties@^8.2.0:
+ version "8.2.0"
+ resolved "https://registry.yarnpkg.com/regenerate-unicode-properties/-/regenerate-unicode-properties-8.2.0.tgz#e5de7111d655e7ba60c057dbe9ff37c87e65cdec"
+ integrity sha512-F9DjY1vKLo/tPePDycuH3dn9H1OTPIkVD9Kz4LODu+F2C75mgjAJ7x/gwy6ZcSNRAAkhNlJSOHRe8k3p+K9WhA==
dependencies:
regenerate "^1.4.0"
@@ -9450,12 +11064,18 @@
resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.3.tgz#7cf6a77d8f5c6f60eb73c5fc1955b2ceb01e6bf5"
integrity sha512-naKIZz2GQ8JWh///G7L3X6LaQUAMp2lvb1rvwwsURe/VXwD6VMfr+/1NuNw3ag8v2kY1aQ/go5SNn79O9JU7yw==
-regenerator-transform@^0.14.0:
- version "0.14.1"
- resolved "https://registry.yarnpkg.com/regenerator-transform/-/regenerator-transform-0.14.1.tgz#3b2fce4e1ab7732c08f665dfdb314749c7ddd2fb"
- integrity sha512-flVuee02C3FKRISbxhXl9mGzdbWUVHubl1SMaknjxkFB1/iqpJhArQUvRxOOPEc/9tAiX0BaQ28FJH10E4isSQ==
+regenerator-runtime@^0.13.4:
+ version "0.13.5"
+ resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.5.tgz#d878a1d094b4306d10b9096484b33ebd55e26697"
+ integrity sha512-ZS5w8CpKFinUzOwW3c83oPeVXoNsrLsaCoLtJvAClH135j/R77RuymhiSErhm2lKcwSCIpmvIWSbDkIfAqKQlA==
+
+regenerator-transform@^0.14.2:
+ version "0.14.4"
+ resolved "https://registry.yarnpkg.com/regenerator-transform/-/regenerator-transform-0.14.4.tgz#5266857896518d1616a78a0479337a30ea974cc7"
+ integrity sha512-EaJaKPBI9GvKpvUz2mz4fhx7WPgvwRLY9v3hlNHWmAuJHI13T4nwKnNvm5RWJzEdnI5g5UwtOww+S8IdoUC2bw==
dependencies:
- private "^0.1.6"
+ "@babel/runtime" "^7.8.4"
+ private "^0.1.8"
regex-not@^1.0.0, regex-not@^1.0.2:
version "1.0.2"
@@ -9465,7 +11085,17 @@
extend-shallow "^3.0.2"
safe-regex "^1.1.0"
-regexp.prototype.flags@^1.2.0:
+regex-parser@2.2.10:
+ version "2.2.10"
+ resolved "https://registry.yarnpkg.com/regex-parser/-/regex-parser-2.2.10.tgz#9e66a8f73d89a107616e63b39d4deddfee912b37"
+ integrity sha512-8t6074A68gHfU8Neftl0Le6KTDwfGAj7IyjPIMSfikI2wJUTHDMaIq42bUsfVnj8mhx0R+45rdUXHGpN164avA==
+
+regexp-tree@^0.1.21, regexp-tree@~0.1.1:
+ version "0.1.21"
+ resolved "https://registry.yarnpkg.com/regexp-tree/-/regexp-tree-0.1.21.tgz#55e2246b7f7d36f1b461490942fa780299c400d7"
+ integrity sha512-kUUXjX4AnqnR8KRTCrayAo9PzYMRKmVoGgaz2tBuz0MF3g1ZbGebmtW0yFHfFK9CmBjQKeYIgoL22pFLBJY7sw==
+
+regexp.prototype.flags@^1.2.0, regexp.prototype.flags@^1.3.0:
version "1.3.0"
resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.3.0.tgz#7aba89b3c13a64509dabcf3ca8d9fbb9bdf5cb75"
integrity sha512-2+Q0C5g951OlYlJz6yu5/M33IcsESLlLfsyIaLJaG4FA2r4yP8MvVMJUUP/fVBkSpbbbZlS5gynbEWLipiiXiQ==
@@ -9478,17 +11108,22 @@
resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-2.0.1.tgz#8d19d31cf632482b589049f8281f93dbcba4d07f"
integrity sha512-lv0M6+TkDVniA3aD1Eg0DVpfU/booSu7Eev3TDO/mZKHBfVjgCGTV4t4buppESEYDtkArYFOxTJWv6S5C+iaNw==
-regexpu-core@^4.6.0:
- version "4.6.0"
- resolved "https://registry.yarnpkg.com/regexpu-core/-/regexpu-core-4.6.0.tgz#2037c18b327cfce8a6fea2a4ec441f2432afb8b6"
- integrity sha512-YlVaefl8P5BnFYOITTNzDvan1ulLOiXJzCNZxduTIosN17b87h3bvG9yHMoHaRuo88H4mQ06Aodj5VtYGGGiTg==
+regexpp@^3.0.0:
+ version "3.1.0"
+ resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-3.1.0.tgz#206d0ad0a5648cffbdb8ae46438f3dc51c9f78e2"
+ integrity sha512-ZOIzd8yVsQQA7j8GCSlPGXwg5PfmA1mrq0JP4nGhh54LaKN3xdai/vHUDu74pKwV8OxseMS65u2NImosQcSD0Q==
+
+regexpu-core@^4.7.0:
+ version "4.7.0"
+ resolved "https://registry.yarnpkg.com/regexpu-core/-/regexpu-core-4.7.0.tgz#fcbf458c50431b0bb7b45d6967b8192d91f3d938"
+ integrity sha512-TQ4KXRnIn6tz6tjnrXEkD/sshygKH/j5KzK86X8MkeHyZ8qst/LZ89j3X4/8HEIfHANTFIP/AbXakeRhWIl5YQ==
dependencies:
regenerate "^1.4.0"
- regenerate-unicode-properties "^8.1.0"
- regjsgen "^0.5.0"
- regjsparser "^0.6.0"
+ regenerate-unicode-properties "^8.2.0"
+ regjsgen "^0.5.1"
+ regjsparser "^0.6.4"
unicode-match-property-ecmascript "^1.0.4"
- unicode-match-property-value-ecmascript "^1.1.0"
+ unicode-match-property-value-ecmascript "^1.2.0"
registry-auth-token@^4.0.0:
version "4.0.0"
@@ -9505,19 +11140,19 @@
dependencies:
rc "^1.2.8"
-regjsgen@^0.5.0:
+regjsgen@^0.5.1:
version "0.5.1"
resolved "https://registry.yarnpkg.com/regjsgen/-/regjsgen-0.5.1.tgz#48f0bf1a5ea205196929c0d9798b42d1ed98443c"
integrity sha512-5qxzGZjDs9w4tzT3TPhCJqWdCc3RLYwy9J2NB0nm5Lz+S273lvWcpjaTGHsT1dc6Hhfq41uSEOw8wBmxrKOuyg==
-regjsparser@^0.6.0:
- version "0.6.2"
- resolved "https://registry.yarnpkg.com/regjsparser/-/regjsparser-0.6.2.tgz#fd62c753991467d9d1ffe0a9f67f27a529024b96"
- integrity sha512-E9ghzUtoLwDekPT0DYCp+c4h+bvuUpe6rRHCTYn6eGoqj1LgKXxT6I0Il4WbjhQkOghzi/V+y03bPKvbllL93Q==
+regjsparser@^0.6.4:
+ version "0.6.4"
+ resolved "https://registry.yarnpkg.com/regjsparser/-/regjsparser-0.6.4.tgz#a769f8684308401a66e9b529d2436ff4d0666272"
+ integrity sha512-64O87/dPDgfk8/RQqC4gkZoGyyWFIEUTTh80CU6CWuK5vkCGyekIx+oKcEIYtP/RAxSQltCZHCNu/mdd7fqlJw==
dependencies:
jsesc "~0.5.0"
-relateurl@0.2.x:
+relateurl@^0.2.7:
version "0.2.7"
resolved "https://registry.yarnpkg.com/relateurl/-/relateurl-0.2.7.tgz#54dbf377e51440aca90a4cd274600d3ff2d888a9"
integrity sha1-VNvzd+UUQKypCkzSdGANP/LYiKk=
@@ -9605,16 +11240,16 @@
resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-2.0.0.tgz#d0b329ecc7cc0f61649f62215be69af54aa8989b"
integrity sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==
-requireindex@^1.2.0:
- version "1.2.0"
- resolved "https://registry.yarnpkg.com/requireindex/-/requireindex-1.2.0.tgz#3463cdb22ee151902635aa6c9535d4de9c2ef1ef"
- integrity sha512-L9jEkOi3ASd9PYit2cwRfyppc9NoABujTP8/5gFcbERmo5jUoAKovIC3fsF17pkTnGsrByysqX+Kxd2OTNI1ww==
-
requires-port@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff"
integrity sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8=
+reserved-words@^0.1.2:
+ version "0.1.2"
+ resolved "https://registry.yarnpkg.com/reserved-words/-/reserved-words-0.1.2.tgz#00a0940f98cd501aeaaac316411d9adc52b31ab1"
+ integrity sha1-AKCUD5jNUBrqqsMWQR2a3FKzGrE=
+
resize-observer-polyfill@^1.5.0, resize-observer-polyfill@^1.5.1:
version "1.5.1"
resolved "https://registry.yarnpkg.com/resize-observer-polyfill/-/resize-observer-polyfill-1.5.1.tgz#0e9020dd3d21024458d4ebd27e23e40269810464"
@@ -9627,6 +11262,13 @@
dependencies:
resolve-from "^3.0.0"
+resolve-cwd@^3.0.0:
+ version "3.0.0"
+ resolved "https://registry.yarnpkg.com/resolve-cwd/-/resolve-cwd-3.0.0.tgz#0f0075f1bb2544766cf73ba6a6e2adfebcb13f2d"
+ integrity sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==
+ dependencies:
+ resolve-from "^5.0.0"
+
resolve-from@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-3.0.0.tgz#b22c7af7d9d6881bc8b6e653335eebcb0a188748"
@@ -9637,11 +11279,32 @@
resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6"
integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==
+resolve-from@^5.0.0:
+ version "5.0.0"
+ resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-5.0.0.tgz#c35225843df8f776df21c57557bc087e9dfdfc69"
+ integrity sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==
+
resolve-pathname@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/resolve-pathname/-/resolve-pathname-3.0.0.tgz#99d02224d3cf263689becbb393bc560313025dcd"
integrity sha512-C7rARubxI8bXFNB/hqcp/4iUeIXJhJZvFPFPiSPRnhU5UPxzMFIl+2E6yY6c4k9giDJAhtV+enfA+G89N6Csng==
+resolve-url-loader@3.1.1:
+ version "3.1.1"
+ resolved "https://registry.yarnpkg.com/resolve-url-loader/-/resolve-url-loader-3.1.1.tgz#28931895fa1eab9be0647d3b2958c100ae3c0bf0"
+ integrity sha512-K1N5xUjj7v0l2j/3Sgs5b8CjrrgtC70SmdCuZiJ8tSyb5J+uk3FoeZ4b7yTnH6j7ngI+Bc5bldHJIa8hYdu2gQ==
+ dependencies:
+ adjust-sourcemap-loader "2.0.0"
+ camelcase "5.3.1"
+ compose-function "3.0.3"
+ convert-source-map "1.7.0"
+ es6-iterator "2.0.3"
+ loader-utils "1.2.3"
+ postcss "7.0.21"
+ rework "1.0.1"
+ rework-visit "1.0.0"
+ source-map "0.6.1"
+
resolve-url@^0.2.1:
version "0.2.1"
resolved "https://registry.yarnpkg.com/resolve-url/-/resolve-url-0.2.1.tgz#2c637fe77c893afd2a663fe21aa9080068e2052a"
@@ -9652,20 +11315,27 @@
resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.1.7.tgz#203114d82ad2c5ed9e8e0411b3932875e889e97b"
integrity sha1-IDEU2CrSxe2ejgQRs5ModeiJ6Xs=
-resolve@1.10.0:
- version "1.10.0"
- resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.10.0.tgz#3bdaaeaf45cc07f375656dfd2e54ed0810b101ba"
- integrity sha512-3sUr9aq5OfSg2S9pNtPA9hL1FVEAjvfOC4leW0SNf/mpnaakz2a9femSd6LqAww2RaFctwyf1lCqnTHuF1rxDg==
+resolve@1.15.0:
+ version "1.15.0"
+ resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.15.0.tgz#1b7ca96073ebb52e741ffd799f6b39ea462c67f5"
+ integrity sha512-+hTmAldEGE80U2wJJDC1lebb5jWqvTYAfm3YZ1ckk1gBr0MnCqUKlwK1e+anaFljIl+F5tR5IoZcm4ZDA1zMQw==
dependencies:
path-parse "^1.0.6"
-resolve@^1.10.0, resolve@^1.3.2, resolve@^1.5.0, resolve@^1.8.1, resolve@^1.9.0:
+resolve@^1.10.0, resolve@^1.3.2, resolve@^1.5.0, resolve@^1.8.1:
version "1.14.2"
resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.14.2.tgz#dbf31d0fa98b1f29aa5169783b9c290cb865fea2"
integrity sha512-EjlOBLBO1kxsUxsKjLt7TAECyKW6fOh1VRkykQkKGzcBbjjPIxBqGh0jf7GJ3k/f5mxMqW3htMD3WdTUVtW8HQ==
dependencies:
path-parse "^1.0.6"
+resolve@^1.10.1, resolve@^1.12.0, resolve@^1.13.1, resolve@^1.15.1:
+ version "1.17.0"
+ resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.17.0.tgz#b25941b54968231cc2d1bb76a79cb7f2c0bf8444"
+ integrity sha512-ic+7JYiV8Vi2yzQGFWOkiZD5Z9z7O2Zhm9XMaTxdJExKasieFCr+yXZ/WmXsckHiKl12ar0y6XiXDx3m4RHn1w==
+ dependencies:
+ path-parse "^1.0.6"
+
responselike@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/responselike/-/responselike-1.0.2.tgz#918720ef3b631c5642be068f15ade5a46f4ba1e7"
@@ -9673,12 +11343,12 @@
dependencies:
lowercase-keys "^1.0.0"
-restore-cursor@^2.0.0:
- version "2.0.0"
- resolved "https://registry.yarnpkg.com/restore-cursor/-/restore-cursor-2.0.0.tgz#9f7ee287f82fd326d4fd162923d62129eee0dfaf"
- integrity sha1-n37ih/gv0ybU/RYpI9YhKe7g368=
+restore-cursor@^3.1.0:
+ version "3.1.0"
+ resolved "https://registry.yarnpkg.com/restore-cursor/-/restore-cursor-3.1.0.tgz#39f67c54b3a7a58cea5236d95cf0034239631f7e"
+ integrity sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==
dependencies:
- onetime "^2.0.0"
+ onetime "^5.1.0"
signal-exit "^3.0.2"
ret@~0.1.10:
@@ -9686,6 +11356,24 @@
resolved "https://registry.yarnpkg.com/ret/-/ret-0.1.15.tgz#b8a4825d5bdb1fc3f6f53c2bc33f81388681c7bc"
integrity sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==
+retry@^0.12.0:
+ version "0.12.0"
+ resolved "https://registry.yarnpkg.com/retry/-/retry-0.12.0.tgz#1b42a6266a21f07421d1b0b54b7dc167b01c013b"
+ integrity sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=
+
+rework-visit@1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/rework-visit/-/rework-visit-1.0.0.tgz#9945b2803f219e2f7aca00adb8bc9f640f842c9a"
+ integrity sha1-mUWygD8hni96ygCtuLyfZA+ELJo=
+
+rework@1.0.1:
+ version "1.0.1"
+ resolved "https://registry.yarnpkg.com/rework/-/rework-1.0.1.tgz#30806a841342b54510aa4110850cd48534144aa7"
+ integrity sha1-MIBqhBNCtUUQqkEQhQzUhTQUSqc=
+ dependencies:
+ convert-source-map "^0.3.3"
+ css "^2.0.0"
+
rgb-regex@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/rgb-regex/-/rgb-regex-1.0.1.tgz#c0e0d6882df0e23be254a475e8edd41915feaeb1"
@@ -9703,7 +11391,7 @@
dependencies:
glob "^7.1.3"
-rimraf@^2.2.8, rimraf@^2.5.4, rimraf@^2.6.1, rimraf@^2.6.3:
+rimraf@^2.5.4, rimraf@^2.6.3, rimraf@^2.7.1:
version "2.7.1"
resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec"
integrity sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==
@@ -9738,6 +11426,11 @@
dependencies:
is-promise "^2.1.0"
+run-async@^2.4.0:
+ version "2.4.1"
+ resolved "https://registry.yarnpkg.com/run-async/-/run-async-2.4.1.tgz#8440eccf99ea3e70bd409d49aab88e10c189a455"
+ integrity sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==
+
run-queue@^1.0.0, run-queue@^1.0.3:
version "1.0.3"
resolved "https://registry.yarnpkg.com/run-queue/-/run-queue-1.0.3.tgz#e848396f057d223f24386924618e25694161ec47"
@@ -9745,10 +11438,10 @@
dependencies:
aproba "^1.1.1"
-rxjs@^6.4.0:
- version "6.5.4"
- resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-6.5.4.tgz#e0777fe0d184cec7872df147f303572d414e211c"
- integrity sha512-naMQXcgEo3csAEGvw/NydRA0fuS2nDZJiw1YUWFKU7aPPAPGZEsD4Iimit96qwCieH6y614MCLYwdkrWx7z/7Q==
+rxjs@^6.5.3:
+ version "6.5.5"
+ resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-6.5.5.tgz#c5c884e3094c8cfee31bf27eb87e54ccfc87f9ec"
+ integrity sha512-WfQI+1gohdf0Dai/Bbmk5L5ItH5tYqm3ki2c5GdWhKjalzjg93N3avFjVStyZZz+A2Em+ZxKH5bNghw9UeylGQ==
dependencies:
tslib "^1.9.0"
@@ -9769,6 +11462,13 @@
dependencies:
ret "~0.1.10"
+safe-regex@^2.1.1:
+ version "2.1.1"
+ resolved "https://registry.yarnpkg.com/safe-regex/-/safe-regex-2.1.1.tgz#f7128f00d056e2fe5c11e81a1324dd974aadced2"
+ integrity sha512-rx+x8AMzKb5Q5lQ95Zoi6ZbJqwCLkqi3XuJXp5P3rT8OEc6sZCJG5AE5dU3lsgRr/F4Bs31jSlVN+j5KrsGu9A==
+ dependencies:
+ regexp-tree "~0.1.1"
+
"safer-buffer@>= 2.1.2 < 3", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0:
version "2.1.2"
resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a"
@@ -9789,17 +11489,21 @@
minimist "^1.1.1"
walker "~1.0.5"
-sass-loader@7.1.0:
- version "7.1.0"
- resolved "https://registry.yarnpkg.com/sass-loader/-/sass-loader-7.1.0.tgz#16fd5138cb8b424bf8a759528a1972d72aad069d"
- integrity sha512-+G+BKGglmZM2GUSfT9TLuEp6tzehHPjAMoRRItOojWIqIGPloVCMhNIQuG639eJ+y033PaGTSjLaTHts8Kw79w==
+sanitize.css@^10.0.0:
+ version "10.0.0"
+ resolved "https://registry.yarnpkg.com/sanitize.css/-/sanitize.css-10.0.0.tgz#b5cb2547e96d8629a60947544665243b1dc3657a"
+ integrity sha512-vTxrZz4dX5W86M6oVWVdOVe72ZiPs41Oi7Z6Km4W5Turyz28mrXSJhhEBZoRtzJWIv3833WKVwLSDWWkEfupMg==
+
+sass-loader@8.0.2:
+ version "8.0.2"
+ resolved "https://registry.yarnpkg.com/sass-loader/-/sass-loader-8.0.2.tgz#debecd8c3ce243c76454f2e8290482150380090d"
+ integrity sha512-7o4dbSK8/Ol2KflEmSco4jTjQoV988bM82P9CZdmo9hR3RLnvNc0ufMNdMrB0caq38JQ/FgF4/7RcbcfKzxoFQ==
dependencies:
- clone-deep "^2.0.1"
- loader-utils "^1.0.1"
- lodash.tail "^4.1.1"
- neo-async "^2.5.0"
- pify "^3.0.0"
- semver "^5.5.0"
+ clone-deep "^4.0.1"
+ loader-utils "^1.2.3"
+ neo-async "^2.6.1"
+ schema-utils "^2.6.1"
+ semver "^6.3.0"
sax@^1.2.4, sax@~1.2.4:
version "1.2.4"
@@ -9830,12 +11534,20 @@
ajv-errors "^1.0.0"
ajv-keywords "^3.1.0"
+schema-utils@^2.5.0, schema-utils@^2.6.0, schema-utils@^2.6.1, schema-utils@^2.6.4, schema-utils@^2.6.5:
+ version "2.6.6"
+ resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-2.6.6.tgz#299fe6bd4a3365dc23d99fd446caff8f1d6c330c"
+ integrity sha512-wHutF/WPSbIi9x6ctjGGk2Hvl0VOz5l3EKEuKbjPlB30mKZUzb9A5k9yEXRX3pwyqVLPvpfZZEllaFq/M718hA==
+ dependencies:
+ ajv "^6.12.0"
+ ajv-keywords "^3.4.1"
+
select-hose@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/select-hose/-/select-hose-2.0.0.tgz#625d8658f865af43ec962bfc376a37359a4994ca"
integrity sha1-Yl2GWPhlr0Psliv8N2o3NZpJlMo=
-selfsigned@^1.9.1:
+selfsigned@^1.10.7:
version "1.10.7"
resolved "https://registry.yarnpkg.com/selfsigned/-/selfsigned-1.10.7.tgz#da5819fd049d5574f28e88a9bcc6dbc6e6f3906b"
integrity sha512-8M3wBCzeWIJnQfl43IKwOmC4H/RAp50S8DF60znzjW5GVqTcSe2vWclt7hmYVPkKPlHWOu5EaWOMZ2Y6W8ZXTA==
@@ -9854,30 +11566,32 @@
dependencies:
semver "^5.0.3"
-"semver@2 || 3 || 4 || 5", semver@^5.0.3, semver@^5.4.1, semver@^5.5.0, semver@^5.5.1, semver@^5.6.0:
+semver-diff@^3.1.1:
+ version "3.1.1"
+ resolved "https://registry.yarnpkg.com/semver-diff/-/semver-diff-3.1.1.tgz#05f77ce59f325e00e2706afd67bb506ddb1ca32b"
+ integrity sha512-GX0Ix/CJcHyB8c4ykpHGIAvLyOwOobtM/8d+TQkAd81/bEjgPHrfba41Vpesr7jX/t8Uh+R3EX9eAS5be+jQYg==
+ dependencies:
+ semver "^6.3.0"
+
+"semver@2 || 3 || 4 || 5", semver@^5.0.3, semver@^5.4.1, semver@^5.5.0, semver@^5.5.1, semver@^5.6.0, semver@^5.7.1:
version "5.7.1"
resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7"
integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==
-semver@5.5.0:
- version "5.5.0"
- resolved "https://registry.yarnpkg.com/semver/-/semver-5.5.0.tgz#dc4bbc7a6ca9d916dee5d43516f0092b58f7b8ab"
- integrity sha512-4SJ3dm0WAwWy/NVeioZh5AntkdJoWKxHxcmyP622fOkgHa4z3R0TdBJICINyaSDE6uNwVc8gZr+ZinwZAH4xIA==
-
-semver@6.0.0:
- version "6.0.0"
- resolved "https://registry.yarnpkg.com/semver/-/semver-6.0.0.tgz#05e359ee571e5ad7ed641a6eec1e547ba52dea65"
- integrity sha512-0UewU+9rFapKFnlbirLi3byoOuhrSsli/z/ihNnvM24vgF+8sNBiI1LZPBSH9wJKUwaUbw+s3hToDLCXkrghrQ==
+semver@6.3.0, semver@^6.0.0, semver@^6.1.0, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0:
+ version "6.3.0"
+ resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d"
+ integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==
semver@7.0.0:
version "7.0.0"
resolved "https://registry.yarnpkg.com/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e"
integrity sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A==
-semver@^6.0.0, semver@^6.2.0, semver@^6.3.0:
- version "6.3.0"
- resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d"
- integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==
+semver@^7.1.3, semver@^7.3.2:
+ version "7.3.2"
+ resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.2.tgz#604962b052b81ed0786aae84389ffba70ffd3938"
+ integrity sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==
send@0.17.1:
version "0.17.1"
@@ -9898,17 +11612,12 @@
range-parser "~1.2.1"
statuses "~1.5.0"
-serialize-javascript@^1.4.0:
- version "1.9.1"
- resolved "https://registry.yarnpkg.com/serialize-javascript/-/serialize-javascript-1.9.1.tgz#cfc200aef77b600c47da9bb8149c943e798c2fdb"
- integrity sha512-0Vb/54WJ6k5v8sSWN09S0ora+Hnr+cX40r9F170nT+mSkaxltoE/7R3OrIdBSUv1OoiobH1QoWQbCnAO+e8J1A==
-
serialize-javascript@^2.1.2:
version "2.1.2"
resolved "https://registry.yarnpkg.com/serialize-javascript/-/serialize-javascript-2.1.2.tgz#ecec53b0e0317bdc95ef76ab7074b7384785fa61"
integrity sha512-rs9OggEUF0V4jUSecXazOYsLfu7OGK2qIn3c7IPBiffz32XniEp/TX9Xmc9LQfK2nQ2QKHvZ2oygKUGU0lG4jQ==
-serve-index@^1.7.2:
+serve-index@^1.9.1:
version "1.9.1"
resolved "https://registry.yarnpkg.com/serve-index/-/serve-index-1.9.1.tgz#d3768d69b1e7d82e5ce050fff5b453bea12a9239"
integrity sha1-03aNabHn2C5c4FD/9bRTvqEqkjk=
@@ -9984,14 +11693,12 @@
lazy-cache "^0.2.3"
mixin-object "^2.0.1"
-shallow-clone@^1.0.0:
- version "1.0.0"
- resolved "https://registry.yarnpkg.com/shallow-clone/-/shallow-clone-1.0.0.tgz#4480cd06e882ef68b2ad88a3ea54832e2c48b571"
- integrity sha512-oeXreoKR/SyNJtRJMAKPDSvd28OqEwG4eR/xc856cRGBII7gX9lvAqDxusPm0846z/w/hWYjI1NpKwJ00NHzRA==
+shallow-clone@^3.0.0:
+ version "3.0.1"
+ resolved "https://registry.yarnpkg.com/shallow-clone/-/shallow-clone-3.0.1.tgz#8f2981ad92531f55035b01fb230769a40e02efa3"
+ integrity sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==
dependencies:
- is-extendable "^0.1.1"
- kind-of "^5.0.0"
- mixin-object "^2.0.1"
+ kind-of "^6.0.2"
shallow-equal@^1.0.0:
version "1.2.1"
@@ -10010,11 +11717,23 @@
dependencies:
shebang-regex "^1.0.0"
+shebang-command@^2.0.0:
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-2.0.0.tgz#ccd0af4f8835fbdc265b82461aaf0c36663f34ea"
+ integrity sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==
+ dependencies:
+ shebang-regex "^3.0.0"
+
shebang-regex@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-1.0.0.tgz#da42f49740c0b42db2ca9728571cb190c98efea3"
integrity sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=
+shebang-regex@^3.0.0:
+ version "3.0.0"
+ resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172"
+ integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==
+
shell-quote@1.7.2, shell-quote@^1.6.1:
version "1.7.2"
resolved "https://registry.yarnpkg.com/shell-quote/-/shell-quote-1.7.2.tgz#67a7d02c76c9da24f99d20808fcaded0e0e04be2"
@@ -10025,6 +11744,14 @@
resolved "https://registry.yarnpkg.com/shellwords/-/shellwords-0.1.1.tgz#d6b9181c1a48d397324c84871efbcfc73fc0654b"
integrity sha512-vFwSUfQvqybiICwZY5+DAWIPLKsWO31Q91JSKl3UYv+K5c2QRPzn0qzec6QPu1Qc9eHYItiP3NdJqNVqetYAww==
+side-channel@^1.0.2:
+ version "1.0.2"
+ resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.2.tgz#df5d1abadb4e4bf4af1cd8852bf132d2f7876947"
+ integrity sha512-7rL9YlPHg7Ancea1S96Pa8/QWb4BtXL/TZvS6B8XFetGBeuhAsfmUspK6DokBeZ64+Kj9TCNRD/30pVz1BvQNA==
+ dependencies:
+ es-abstract "^1.17.0-next.1"
+ object-inspect "^1.7.0"
+
signal-exit@^3.0.0, signal-exit@^3.0.2:
version "3.0.2"
resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.2.tgz#b5fdc08f1287ea1178628e415e25132b73646c6d"
@@ -10052,6 +11779,11 @@
resolved "https://registry.yarnpkg.com/slash/-/slash-2.0.0.tgz#de552851a1759df3a8f206535442f5ec4ddeab44"
integrity sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==
+slash@^3.0.0:
+ version "3.0.0"
+ resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634"
+ integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==
+
slice-ansi@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-2.1.0.tgz#cacd7693461a637a5788d92a7dd4fba068e81636"
@@ -10091,18 +11823,6 @@
source-map-resolve "^0.5.0"
use "^3.1.0"
-sockjs-client@1.3.0:
- version "1.3.0"
- resolved "https://registry.yarnpkg.com/sockjs-client/-/sockjs-client-1.3.0.tgz#12fc9d6cb663da5739d3dc5fb6e8687da95cb177"
- integrity sha512-R9jxEzhnnrdxLCNln0xg5uGHqMnkhPSTzUZH2eXcR03S/On9Yvoq2wyUZILRUhZCNVu2PmwWVoyuiPz8th8zbg==
- dependencies:
- debug "^3.2.5"
- eventsource "^1.0.7"
- faye-websocket "~0.11.1"
- inherits "^2.0.3"
- json3 "^3.3.2"
- url-parse "^1.4.3"
-
sockjs-client@1.4.0:
version "1.4.0"
resolved "https://registry.yarnpkg.com/sockjs-client/-/sockjs-client-1.4.0.tgz#c9f2568e19c8fd8173b4997ea3420e0bb306c7d5"
@@ -10123,12 +11843,19 @@
faye-websocket "^0.10.0"
uuid "^3.0.1"
+sort-keys@^1.0.0:
+ version "1.1.2"
+ resolved "https://registry.yarnpkg.com/sort-keys/-/sort-keys-1.1.2.tgz#441b6d4d346798f1b4e49e8920adfba0e543f9ad"
+ integrity sha1-RBttTTRnmPG05J6JIK37oOVD+a0=
+ dependencies:
+ is-plain-obj "^1.0.0"
+
source-list-map@^2.0.0:
version "2.0.1"
resolved "https://registry.yarnpkg.com/source-list-map/-/source-list-map-2.0.1.tgz#3993bd873bfc48479cca9ea3a547835c7c154b34"
integrity sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw==
-source-map-resolve@^0.5.0:
+source-map-resolve@^0.5.0, source-map-resolve@^0.5.2:
version "0.5.3"
resolved "https://registry.yarnpkg.com/source-map-resolve/-/source-map-resolve-0.5.3.tgz#190866bece7553e1f8f267a2ee82c606b5509a1a"
integrity sha512-Htz+RnsXWk5+P2slx5Jh3Q66vhQj1Cllm0zvnaY98+NFx+Dv2CF/f5O/t8x+KaNdrdIAsruNzoh/KpialbqAnw==
@@ -10139,7 +11866,7 @@
source-map-url "^0.4.0"
urix "^0.1.0"
-source-map-support@^0.5.6, source-map-support@~0.5.10, source-map-support@~0.5.12:
+source-map-support@^0.5.6, source-map-support@~0.5.12:
version "0.5.16"
resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.16.tgz#0ae069e7fe3ba7538c64c98515e35339eac5a042"
integrity sha512-efyLRJDr68D9hBBNIPWFjhpFzURh+KJykQwvMyW5UiZzYwoF6l4YMMDIJJEyFWxWCqfyxLzz6tSfUFR+kXXsVQ==
@@ -10152,16 +11879,16 @@
resolved "https://registry.yarnpkg.com/source-map-url/-/source-map-url-0.4.0.tgz#3e935d7ddd73631b97659956d55128e87b5084a3"
integrity sha1-PpNdfd1zYxuXZZlW1VEo6HtQhKM=
+source-map@0.6.1, source-map@^0.6.0, source-map@^0.6.1, source-map@~0.6.0, source-map@~0.6.1:
+ version "0.6.1"
+ resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263"
+ integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==
+
source-map@^0.5.0, source-map@^0.5.6:
version "0.5.7"
resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc"
integrity sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=
-source-map@^0.6.0, source-map@^0.6.1, source-map@~0.6.0, source-map@~0.6.1:
- version "0.6.1"
- resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263"
- integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==
-
spdx-correct@^3.0.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/spdx-correct/-/spdx-correct-3.1.0.tgz#fb83e504445268f154b074e218c87c003cd31df4"
@@ -10200,10 +11927,10 @@
readable-stream "^3.0.6"
wbuf "^1.7.3"
-spdy@^4.0.0:
- version "4.0.1"
- resolved "https://registry.yarnpkg.com/spdy/-/spdy-4.0.1.tgz#6f12ed1c5db7ea4f24ebb8b89ba58c87c08257f2"
- integrity sha512-HeZS3PBdMA+sZSu0qwpCxl3DeALD5ASx8pAX0jZdKXSpPWbQ6SYGnlg3BBmYLx5LtiZrmkAZfErCm2oECBcioA==
+spdy@^4.0.1:
+ version "4.0.2"
+ resolved "https://registry.yarnpkg.com/spdy/-/spdy-4.0.2.tgz#b74f466203a3eda452c02492b91fb9e84a27677b"
+ integrity sha512-r46gZQZQV+Kl9oItvl1JZZqJKGr+oEkB08A6BzkiR7593/7IbtuncXHd2YoYeTsG4157ZssMu9KYvUHLcjcDoA==
dependencies:
debug "^4.1.0"
handle-thing "^2.0.0"
@@ -10245,6 +11972,14 @@
dependencies:
figgy-pudding "^3.5.1"
+ssri@^7.0.0:
+ version "7.1.0"
+ resolved "https://registry.yarnpkg.com/ssri/-/ssri-7.1.0.tgz#92c241bf6de82365b5c7fb4bd76e975522e1294d"
+ integrity sha512-77/WrDZUWocK0mvA5NTRQyveUf+wsrIc6vyrxpS8tVvYBcX215QbafrJR3KtkpskIzoFLqqNuuYQvxaMjXJ/0g==
+ dependencies:
+ figgy-pudding "^3.5.1"
+ minipass "^3.1.1"
+
stable@^0.1.8:
version "0.1.8"
resolved "https://registry.yarnpkg.com/stable/-/stable-0.1.8.tgz#836eb3c8382fe2936feaf544631017ce7d47a3cf"
@@ -10312,6 +12047,11 @@
resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.1.tgz#d7088281559ab2778424279b0877da3c392d5a3d"
integrity sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==
+strict-uri-encode@^1.0.0:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz#279b225df1d582b1f54e65addd4352e18faa0713"
+ integrity sha1-J5siXfHVgrH1TmWt3UNS4Y+qBxM=
+
string-convert@^0.2.0:
version "0.2.1"
resolved "https://registry.yarnpkg.com/string-convert/-/string-convert-0.2.1.tgz#6982cc3049fbb4cd85f8b24568b9d9bf39eeff97"
@@ -10325,6 +12065,14 @@
astral-regex "^1.0.0"
strip-ansi "^4.0.0"
+string-length@^3.1.0:
+ version "3.1.0"
+ resolved "https://registry.yarnpkg.com/string-length/-/string-length-3.1.0.tgz#107ef8c23456e187a8abd4a61162ff4ac6e25837"
+ integrity sha512-Ttp5YvkGm5v9Ijagtaz1BnN+k9ObpvS0eIBblPMp2YWL8FBmi9qblQ9fexc2k/CXFgrTIteU3jAw3payCnwSTA==
+ dependencies:
+ astral-regex "^1.0.0"
+ strip-ansi "^5.2.0"
+
string-width@^1.0.1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/string-width/-/string-width-1.0.2.tgz#118bdf5b8cdc51a2a7e70d211e07e2b0b9b107d3"
@@ -10334,7 +12082,7 @@
is-fullwidth-code-point "^1.0.0"
strip-ansi "^3.0.0"
-string-width@^2.0.0, string-width@^2.1.0, string-width@^2.1.1:
+string-width@^2.0.0, string-width@^2.1.1:
version "2.1.1"
resolved "https://registry.yarnpkg.com/string-width/-/string-width-2.1.1.tgz#ab93f27a8dc13d28cac815c462143a6d9012ae9e"
integrity sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==
@@ -10351,6 +12099,27 @@
is-fullwidth-code-point "^2.0.0"
strip-ansi "^5.1.0"
+string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0:
+ version "4.2.0"
+ resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.0.tgz#952182c46cc7b2c313d1596e623992bd163b72b5"
+ integrity sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==
+ dependencies:
+ emoji-regex "^8.0.0"
+ is-fullwidth-code-point "^3.0.0"
+ strip-ansi "^6.0.0"
+
+string.prototype.matchall@^4.0.2:
+ version "4.0.2"
+ resolved "https://registry.yarnpkg.com/string.prototype.matchall/-/string.prototype.matchall-4.0.2.tgz#48bb510326fb9fdeb6a33ceaa81a6ea04ef7648e"
+ integrity sha512-N/jp6O5fMf9os0JU3E72Qhf590RSRZU/ungsL/qJUYVTNv7hTG0P/dbPjxINVN9jpscu3nzYwKESU3P3RY5tOg==
+ dependencies:
+ define-properties "^1.1.3"
+ es-abstract "^1.17.0"
+ has-symbols "^1.0.1"
+ internal-slot "^1.0.2"
+ regexp.prototype.flags "^1.3.0"
+ side-channel "^1.0.2"
+
string.prototype.padend@^3.0.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/string.prototype.padend/-/string.prototype.padend-3.1.0.tgz#dc08f57a8010dc5c153550318f67e13adbb72ac3"
@@ -10398,12 +12167,12 @@
is-obj "^1.0.1"
is-regexp "^1.0.0"
-strip-ansi@5.2.0, strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0:
- version "5.2.0"
- resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-5.2.0.tgz#8c9a536feb6afc962bdfa5b104a5091c1ad9c0ae"
- integrity sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==
+strip-ansi@6.0.0, strip-ansi@^6.0.0:
+ version "6.0.0"
+ resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.0.tgz#0b1571dd7669ccd4f3e06e14ef1eed26225ae532"
+ integrity sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==
dependencies:
- ansi-regex "^4.1.0"
+ ansi-regex "^5.0.0"
strip-ansi@^3.0.0, strip-ansi@^3.0.1:
version "3.0.1"
@@ -10419,6 +12188,13 @@
dependencies:
ansi-regex "^3.0.0"
+strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0:
+ version "5.2.0"
+ resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-5.2.0.tgz#8c9a536feb6afc962bdfa5b104a5091c1ad9c0ae"
+ integrity sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==
+ dependencies:
+ ansi-regex "^4.1.0"
+
strip-bom@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-3.0.0.tgz#2334c18e9c759f7bdd56fdef7e9ae3d588e68ed3"
@@ -10437,7 +12213,17 @@
resolved "https://registry.yarnpkg.com/strip-eof/-/strip-eof-1.0.0.tgz#bb43ff5598a6eb05d89b59fcd129c983313606bf"
integrity sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=
-strip-json-comments@^2.0.1, strip-json-comments@~2.0.1:
+strip-indent@^2.0.0:
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/strip-indent/-/strip-indent-2.0.0.tgz#5ef8db295d01e6ed6cbf7aab96998d7822527b68"
+ integrity sha1-XvjbKV0B5u1sv3qrlpmNeCJSe2g=
+
+strip-json-comments@^3.0.1:
+ version "3.1.0"
+ resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.0.tgz#7638d31422129ecf4457440009fba03f9f9ac180"
+ integrity sha512-e6/d0eBu7gHtdCqFt0xJr642LdToM5/cN4Qb9DbHjVx1CP5RyeM+zH7pbecEmDv/lBqb0QH+6Uqq75rxFPkM0w==
+
+strip-json-comments@~2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a"
integrity sha1-PFMZQukIwml8DsNEhYwobHygpgo=
@@ -10478,6 +12264,21 @@
dependencies:
has-flag "^3.0.0"
+supports-color@^7.0.0, supports-color@^7.1.0:
+ version "7.1.0"
+ resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.1.0.tgz#68e32591df73e25ad1c4b49108a2ec507962bfd1"
+ integrity sha512-oRSIpR8pxT1Wr2FquTNnGet79b3BWljqOuoW/h4oBhxJ/HUbX5nX6JSruTkvXDCFMwDPvsaTTbvMLKZWSy0R5g==
+ dependencies:
+ has-flag "^4.0.0"
+
+supports-hyperlinks@^2.0.0:
+ version "2.1.0"
+ resolved "https://registry.yarnpkg.com/supports-hyperlinks/-/supports-hyperlinks-2.1.0.tgz#f663df252af5f37c5d49bbd7eeefa9e0b9e59e47"
+ integrity sha512-zoE5/e+dnEijk6ASB6/qrK+oYdm2do1hjoLWrqUC/8WEIW1gbxFcKuBof7sW8ArN6e+AYvsE8HBGiVRWL/F5CA==
+ dependencies:
+ has-flag "^4.0.0"
+ supports-color "^7.0.0"
+
svg-parser@^2.0.0:
version "2.0.2"
resolved "https://registry.yarnpkg.com/svg-parser/-/svg-parser-2.0.2.tgz#d134cc396fa2681dc64f518330784e98bd801ec8"
@@ -10517,7 +12318,12 @@
slice-ansi "^2.1.0"
string-width "^3.0.0"
-tapable@^1.0.0, tapable@^1.1.0:
+tapable@^0.1.8:
+ version "0.1.10"
+ resolved "https://registry.yarnpkg.com/tapable/-/tapable-0.1.10.tgz#29c35707c2b70e50d07482b5d202e8ed446dafd4"
+ integrity sha1-KcNXB8K3DlDQdIK10gLo7URtr9Q=
+
+tapable@^1.0.0, tapable@^1.1.3:
version "1.1.3"
resolved "https://registry.yarnpkg.com/tapable/-/tapable-1.1.3.tgz#a1fccc06b58db61fd7a45da2da44f5f3a3e67ba2"
integrity sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA==
@@ -10529,21 +12335,27 @@
dependencies:
execa "^0.7.0"
-terser-webpack-plugin@1.2.3:
- version "1.2.3"
- resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-1.2.3.tgz#3f98bc902fac3e5d0de730869f50668561262ec8"
- integrity sha512-GOK7q85oAb/5kE12fMuLdn2btOS9OBZn4VsecpHDywoUC/jLhSAKOiYo0ezx7ss2EXPMzyEWFoE0s1WLE+4+oA==
- dependencies:
- cacache "^11.0.2"
- find-cache-dir "^2.0.0"
- schema-utils "^1.0.0"
- serialize-javascript "^1.4.0"
- source-map "^0.6.1"
- terser "^3.16.1"
- webpack-sources "^1.1.0"
- worker-farm "^1.5.2"
+term-size@^2.1.0:
+ version "2.2.0"
+ resolved "https://registry.yarnpkg.com/term-size/-/term-size-2.2.0.tgz#1f16adedfe9bdc18800e1776821734086fcc6753"
+ integrity sha512-a6sumDlzyHVJWb8+YofY4TW112G6p2FCPEAFk+59gIYHv3XHRhm9ltVQ9kli4hNWeQBwSpe8cRN25x0ROunMOw==
-terser-webpack-plugin@^1.1.0:
+terser-webpack-plugin@2.3.5:
+ version "2.3.5"
+ resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-2.3.5.tgz#5ad971acce5c517440ba873ea4f09687de2f4a81"
+ integrity sha512-WlWksUoq+E4+JlJ+h+U+QUzXpcsMSSNXkDy9lBVkSqDn1w23Gg29L/ary9GeJVYCGiNJJX7LnVc4bwL1N3/g1w==
+ dependencies:
+ cacache "^13.0.1"
+ find-cache-dir "^3.2.0"
+ jest-worker "^25.1.0"
+ p-limit "^2.2.2"
+ schema-utils "^2.6.4"
+ serialize-javascript "^2.1.2"
+ source-map "^0.6.1"
+ terser "^4.4.3"
+ webpack-sources "^1.4.3"
+
+terser-webpack-plugin@^1.4.3:
version "1.4.3"
resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-1.4.3.tgz#5ecaf2dbdc5fb99745fd06791f46fc9ddb1c9a7c"
integrity sha512-QMxecFz/gHQwteWwSo5nTc6UaICqN1bMedC5sMtUc7y3Ha3Q8y6ZO0iCR8pq4RJC8Hjf0FEPEHZqcMB/+DFCrA==
@@ -10558,15 +12370,6 @@
webpack-sources "^1.4.0"
worker-farm "^1.7.0"
-terser@^3.16.1:
- version "3.17.0"
- resolved "https://registry.yarnpkg.com/terser/-/terser-3.17.0.tgz#f88ffbeda0deb5637f9d24b0da66f4e15ab10cb2"
- integrity sha512-/FQzzPJmCpjAH9Xvk2paiWrFq+5M6aVOf+2KRbwhByISDX/EujxsK+BAvrhb6H+2rtrLCHK9N01wO014vrIwVQ==
- dependencies:
- commander "^2.19.0"
- source-map "~0.6.1"
- source-map-support "~0.5.10"
-
terser@^4.1.2:
version "4.6.1"
resolved "https://registry.yarnpkg.com/terser/-/terser-4.6.1.tgz#913e35e0d38a75285a7913ba01d753c4089ebdbd"
@@ -10576,6 +12379,15 @@
source-map "~0.6.1"
source-map-support "~0.5.12"
+terser@^4.4.3, terser@^4.6.3:
+ version "4.6.13"
+ resolved "https://registry.yarnpkg.com/terser/-/terser-4.6.13.tgz#e879a7364a5e0db52ba4891ecde007422c56a916"
+ integrity sha512-wMvqukYgVpQlymbnNbabVZbtM6PN63AzqexpwJL8tbh/mRT9LE5o+ruVduAGL7D6Fpjl+Q+06U5I9Ul82odAhw==
+ dependencies:
+ commander "^2.20.0"
+ source-map "~0.6.1"
+ source-map-support "~0.5.12"
+
test-exclude@^5.2.3:
version "5.2.3"
resolved "https://registry.yarnpkg.com/test-exclude/-/test-exclude-5.2.3.tgz#c3d3e1e311eb7ee405e092dac10aefd09091eac0"
@@ -10653,6 +12465,14 @@
resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.4.tgz#23640dd7b42d00433911140820e5cf440e521dd1"
integrity sha1-I2QN17QtAEM5ERQIIOXPRA5SHdE=
+to-absolute-glob@^2.0.2:
+ version "2.0.2"
+ resolved "https://registry.yarnpkg.com/to-absolute-glob/-/to-absolute-glob-2.0.2.tgz#1865f43d9e74b0822db9f145b78cff7d0f7c849b"
+ integrity sha1-GGX0PZ50sIItufFFt4z/fQ98hJs=
+ dependencies:
+ is-absolute "^1.0.0"
+ is-negated-glob "^1.0.0"
+
to-arraybuffer@^1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/to-arraybuffer/-/to-arraybuffer-1.0.1.tgz#7d229b1fcc637e466ca081180836a7aabff83f43"
@@ -10683,6 +12503,13 @@
is-number "^3.0.0"
repeat-string "^1.6.1"
+to-regex-range@^5.0.1:
+ version "5.0.1"
+ resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-5.0.1.tgz#1648c44aae7c8d988a326018ed72f5b4dd0392e4"
+ integrity sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==
+ dependencies:
+ is-number "^7.0.0"
+
to-regex@^3.0.1, to-regex@^3.0.2:
version "3.0.2"
resolved "https://registry.yarnpkg.com/to-regex/-/to-regex-3.0.2.tgz#13cfdd9b336552f30b51f33a8ae1b42a7a7599ce"
@@ -10726,17 +12553,32 @@
dependencies:
punycode "^2.1.0"
-ts-pnp@^1.0.0:
- version "1.1.5"
- resolved "https://registry.yarnpkg.com/ts-pnp/-/ts-pnp-1.1.5.tgz#840e0739c89fce5f3abd9037bb091dbff16d9dec"
- integrity sha512-ti7OGMOUOzo66wLF3liskw6YQIaSsBgc4GOAlWRnIEj8htCxJUxskanMUoJOD6MDCRAXo36goXJZch+nOS0VMA==
+trim-newlines@^2.0.0:
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-2.0.0.tgz#b403d0b91be50c331dfc4b82eeceb22c3de16d20"
+ integrity sha1-tAPQuRvlDDMd/EuC7s6yLD3hbSA=
+
+ts-pnp@1.1.6:
+ version "1.1.6"
+ resolved "https://registry.yarnpkg.com/ts-pnp/-/ts-pnp-1.1.6.tgz#389a24396d425a0d3162e96d2b4638900fdc289a"
+ integrity sha512-CrG5GqAAzMT7144Cl+UIFP7mz/iIhiy+xQ6GGcnjTezhALT02uPMRw7tgDSESgB5MsfKt55+GPWw4ir1kVtMIQ==
+
+ts-pnp@^1.1.6:
+ version "1.2.0"
+ resolved "https://registry.yarnpkg.com/ts-pnp/-/ts-pnp-1.2.0.tgz#a500ad084b0798f1c3071af391e65912c86bca92"
+ integrity sha512-csd+vJOb/gkzvcCHgTGSChYpy5f1/XKNsmvBGO4JXS+z1v2HobugDz4s1IeFXM3wZB44uczs+eazB5Q/ccdhQw==
+
+tslib@^1.10.0:
+ version "1.11.2"
+ resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.11.2.tgz#9c79d83272c9a7aaf166f73915c9667ecdde3cc9"
+ integrity sha512-tTSkux6IGPnUGUd1XAZHcpu85MOkIl5zX49pO+jfsie3eP0B6pyhOlLXm3cAC6T7s+euSDDUUV+Acop5WmtkVg==
tslib@^1.8.1, tslib@^1.9.0:
version "1.10.0"
resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.10.0.tgz#c3c19f95973fb0a62973fb09d90d961ee43e5c8a"
integrity sha512-qOebF53frne81cf0S9B41ByenJ3/IuH8yJKngAX35CmiZySA0khhkovshKK+jGCaMnVomla7gVlIcc3EvKPbTQ==
-tsutils@^3.7.0:
+tsutils@^3.17.1:
version "3.17.1"
resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.17.1.tgz#ed719917f11ca0dee586272b2ac49e015a2dd759"
integrity sha512-kzeQ5B8H3w60nFY2g8cJIuH7JDpsALXySGtwGJ0p2LSjLgay3NdIpqq5SoOBe46bKDW2iq25irHCr8wjomUS2g==
@@ -10767,11 +12609,31 @@
dependencies:
prelude-ls "~1.1.2"
+type-fest@^0.11.0:
+ version "0.11.0"
+ resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.11.0.tgz#97abf0872310fed88a5c466b25681576145e33f1"
+ integrity sha512-OdjXJxnCN1AvyLSzeKIgXTXxV+99ZuXl3Hpo9XpJAv9MBcHrrJOQ5kV7ypXOuQie+AmWG25hLbiKdwYTifzcfQ==
+
type-fest@^0.3.0:
version "0.3.1"
resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.3.1.tgz#63d00d204e059474fe5e1b7c011112bbd1dc29e1"
integrity sha512-cUGJnCdr4STbePCgqNFbpVNCepa+kAVohJs1sLhxzdH+gnEoOd8VhbYa7pD3zZYGiURWM2xzEII3fQcRizDkYQ==
+type-fest@^0.4.1:
+ version "0.4.1"
+ resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.4.1.tgz#8bdf77743385d8a4f13ba95f610f5ccd68c728f8"
+ integrity sha512-IwzA/LSfD2vC1/YDYMv/zHP4rDF1usCwllsDpbolT3D4fUepIO7f9K70jjmUewU/LmGUKJcwcVtDCpnKk4BPMw==
+
+type-fest@^0.6.0:
+ version "0.6.0"
+ resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.6.0.tgz#8d2a2370d3df886eb5c90ada1c5bf6188acf838b"
+ integrity sha512-q+MB8nYR1KDLrgr4G5yemftpMC7/QLqVndBmEEdqzmNj5dcFOO4Oo8qlwZE3ULT3+Zim1F8Kq4cBnikNhlCMlg==
+
+type-fest@^0.8.1:
+ version "0.8.1"
+ resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.8.1.tgz#09e249ebde851d3b1e48d27c105444667f17b83d"
+ integrity sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==
+
type-is@~1.6.17, type-is@~1.6.18:
version "1.6.18"
resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.18.tgz#4e552cd05df09467dcbc4ef739de89f2cf37c131"
@@ -10780,6 +12642,23 @@
media-typer "0.3.0"
mime-types "~2.1.24"
+type@^1.0.1:
+ version "1.2.0"
+ resolved "https://registry.yarnpkg.com/type/-/type-1.2.0.tgz#848dd7698dafa3e54a6c479e759c4bc3f18847a0"
+ integrity sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==
+
+type@^2.0.0:
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/type/-/type-2.0.0.tgz#5f16ff6ef2eb44f260494dae271033b29c09a9c3"
+ integrity sha512-KBt58xCHry4Cejnc2ISQAF7QY+ORngsWfxezO68+12hKV6lQY8P/psIkcbjeHWn7MqcgciWJyCCevFMJdIXpow==
+
+typedarray-to-buffer@^3.1.5:
+ version "3.1.5"
+ resolved "https://registry.yarnpkg.com/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz#a97ee7a9ff42691b9f783ff1bc5112fe3fca9080"
+ integrity sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==
+ dependencies:
+ is-typedarray "^1.0.0"
+
typedarray@^0.0.6:
version "0.0.6"
resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777"
@@ -10790,19 +12669,16 @@
resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.4.5.tgz#2d2618d10bb566572b8d7aad5180d84257d70a99"
integrity sha512-YycBxUb49UUhdNMU5aJ7z5Ej2XGmaIBL0x34vZ82fn3hGvD+bgrMrVDpatgz2f7YxUMJxMkbWxJZeAvDxVe7Vw==
+typescript@^3.0.0:
+ version "3.8.3"
+ resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.8.3.tgz#409eb8544ea0335711205869ec458ab109ee1061"
+ integrity sha512-MYlEfn5VrLNsgudQTVJeNaQFUAI7DkhnOjdpAp4T+ku1TfQClewlbSuTVHiA+8skNBgaf02TL/kLOvig4y3G8w==
+
ua-parser-js@^0.7.18:
version "0.7.21"
resolved "https://registry.yarnpkg.com/ua-parser-js/-/ua-parser-js-0.7.21.tgz#853cf9ce93f642f67174273cc34565ae6f308777"
integrity sha512-+O8/qh/Qj8CgC6eYBVBykMrNtp5Gebn4dlGD/kKXVkJNDwyrAwSIqwz8CDf+tsAIWVycKcku6gIXJ0qwx/ZXaQ==
-uglify-js@3.4.x:
- version "3.4.10"
- resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.4.10.tgz#9ad9563d8eb3acdfb8d38597d2af1d815f6a755f"
- integrity sha512-Y2VsbPVs0FIshJztycsO2SfPk7/KAF/T72qzv9u5EpQ4kB2hQoHlhNQTsNyy6ul7lQtqJN/AoWeS23OzEiEFxw==
- dependencies:
- commander "~2.19.0"
- source-map "~0.6.1"
-
uglify-js@^3.1.4:
version "3.7.4"
resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.7.4.tgz#e6d83a1aa32ff448bd1679359ab13d8db0fe0743"
@@ -10811,6 +12687,11 @@
commander "~2.20.3"
source-map "~0.6.1"
+unc-path-regex@^0.1.2:
+ version "0.1.2"
+ resolved "https://registry.yarnpkg.com/unc-path-regex/-/unc-path-regex-0.1.2.tgz#e73dd3d7b0d7c5ed86fbac6b0ae7d8c6a69d50fa"
+ integrity sha1-5z3T17DXxe2G+6xrCufYxqadUPo=
+
unicode-canonical-property-names-ecmascript@^1.0.4:
version "1.0.4"
resolved "https://registry.yarnpkg.com/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz#2619800c4c825800efdd8343af7dd9933cbe2818"
@@ -10824,10 +12705,10 @@
unicode-canonical-property-names-ecmascript "^1.0.4"
unicode-property-aliases-ecmascript "^1.0.4"
-unicode-match-property-value-ecmascript@^1.1.0:
- version "1.1.0"
- resolved "https://registry.yarnpkg.com/unicode-match-property-value-ecmascript/-/unicode-match-property-value-ecmascript-1.1.0.tgz#5b4b426e08d13a80365e0d657ac7a6c1ec46a277"
- integrity sha512-hDTHvaBk3RmFzvSl0UVrUmC3PuW9wKVnpoUDYH0JDkSIovzw+J5viQmeYHxVSBptubnr7PbH2e0fnpDRQnQl5g==
+unicode-match-property-value-ecmascript@^1.2.0:
+ version "1.2.0"
+ resolved "https://registry.yarnpkg.com/unicode-match-property-value-ecmascript/-/unicode-match-property-value-ecmascript-1.2.0.tgz#0d91f600eeeb3096aa962b1d6fc88876e64ea531"
+ integrity sha512-wjuQHGQVofmSJv1uVISKLE5zO2rNGzM/KCYZch/QQvez7C1hUhBIuZ701fYXExuufJFMPhv2SyL8CyoIfMLbIQ==
unicode-property-aliases-ecmascript@^1.0.4:
version "1.0.5"
@@ -10875,11 +12756,23 @@
dependencies:
crypto-random-string "^1.0.0"
+unique-string@^2.0.0:
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/unique-string/-/unique-string-2.0.0.tgz#39c6451f81afb2749de2b233e3f7c5e8843bd89d"
+ integrity sha512-uNaeirEPvpZWSgzwsPGtU2zVSTrn/8L5q/IexZmH0eH6SA73CmAA5U4GwORTxQAZs95TAXLNqeLoPPNO5gZfWg==
+ dependencies:
+ crypto-random-string "^2.0.0"
+
universalify@^0.1.0:
version "0.1.2"
resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66"
integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==
+universalify@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/universalify/-/universalify-1.0.0.tgz#b61a1da173e8435b2fe3c67d29b9adf8594bd16d"
+ integrity sha512-rb6X1W158d7pRQBg5gkR8uPaSfiids68LTJQYOtEUhoJUWBdaQHsuT/EUduxXYxcrt4r5PJ4fuHW1MHT6p0qug==
+
unpipe@1.0.0, unpipe@~1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec"
@@ -10921,10 +12814,24 @@
semver-diff "^2.0.0"
xdg-basedir "^3.0.0"
-upper-case@^1.1.1:
- version "1.1.3"
- resolved "https://registry.yarnpkg.com/upper-case/-/upper-case-1.1.3.tgz#f6b4501c2ec4cdd26ba78be7222961de77621598"
- integrity sha1-9rRQHC7EzdJrp4vnIilh3ndiFZg=
+update-notifier@^4.0.0:
+ version "4.1.0"
+ resolved "https://registry.yarnpkg.com/update-notifier/-/update-notifier-4.1.0.tgz#4866b98c3bc5b5473c020b1250583628f9a328f3"
+ integrity sha512-w3doE1qtI0/ZmgeoDoARmI5fjDoT93IfKgEGqm26dGUOh8oNpaSTsGNdYRN/SjOuo10jcJGwkEL3mroKzktkew==
+ dependencies:
+ boxen "^4.2.0"
+ chalk "^3.0.0"
+ configstore "^5.0.1"
+ has-yarn "^2.1.0"
+ import-lazy "^2.1.0"
+ is-ci "^2.0.0"
+ is-installed-globally "^0.3.1"
+ is-npm "^4.0.0"
+ is-yarn-global "^0.3.0"
+ latest-version "^5.0.0"
+ pupa "^2.0.1"
+ semver-diff "^3.1.1"
+ xdg-basedir "^4.0.0"
uri-js@^4.2.2:
version "4.2.2"
@@ -10938,14 +12845,14 @@
resolved "https://registry.yarnpkg.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72"
integrity sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI=
-url-loader@1.1.2:
- version "1.1.2"
- resolved "https://registry.yarnpkg.com/url-loader/-/url-loader-1.1.2.tgz#b971d191b83af693c5e3fea4064be9e1f2d7f8d8"
- integrity sha512-dXHkKmw8FhPqu8asTc1puBfe3TehOCo2+RmOOev5suNCIYBcT626kxiWg1NBVkwc4rO8BGa7gP70W7VXuqHrjg==
+url-loader@2.3.0:
+ version "2.3.0"
+ resolved "https://registry.yarnpkg.com/url-loader/-/url-loader-2.3.0.tgz#e0e2ef658f003efb8ca41b0f3ffbf76bab88658b"
+ integrity sha512-goSdg8VY+7nPZKUEChZSEtW5gjbS66USIGCeSJ1OVOJ7Yfuh/36YxCwMi5HVEJh6mqUYOoy3NJ0vlOMrWsSHog==
dependencies:
- loader-utils "^1.1.0"
- mime "^2.0.3"
- schema-utils "^1.0.0"
+ loader-utils "^1.2.3"
+ mime "^2.4.4"
+ schema-utils "^2.5.0"
url-parse-lax@^3.0.0:
version "3.0.0"
@@ -11017,6 +12924,11 @@
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.3.tgz#4568f0216e78760ee1dbf3a4d2cf53e224112866"
integrity sha512-pW0No1RGHgzlpHJO1nsVrHKpOEIxkGg1xB+v0ZmdNH5OAeAwzAVrCnI2/6Mtx+Uys6iaylxa+D3g4j63IKKjSQ==
+v8-compile-cache@^2.0.3:
+ version "2.1.0"
+ resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.1.0.tgz#e14de37b31a6d194f5690d67efc4e7f6fc6ab30e"
+ integrity sha512-usZBT3PW+LOjM25wbqIlZwPeJV+3OSz3M1k1Ws8snlW39dZyYL9lOGC5FgPVHfk0jKmjiDV8Z0mIbVQPiwFs7g==
+
validate-npm-package-license@^3.0.1:
version "3.0.4"
resolved "https://registry.yarnpkg.com/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz#fc91f6b9c7ba15c857f4cb2c5defeec39d4f410a"
@@ -11084,12 +12996,12 @@
dependencies:
loose-envify "^1.0.0"
-watchpack@^1.5.0:
- version "1.6.0"
- resolved "https://registry.yarnpkg.com/watchpack/-/watchpack-1.6.0.tgz#4bc12c2ebe8aa277a71f1d3f14d685c7b446cd00"
- integrity sha512-i6dHe3EyLjMmDlU1/bGQpEw25XSjkJULPuAVKCbNRefQVq48yXKUpwg538F7AZTf9kyr57zj++pQFltUa5H7yA==
+watchpack@^1.6.0:
+ version "1.6.1"
+ resolved "https://registry.yarnpkg.com/watchpack/-/watchpack-1.6.1.tgz#280da0a8718592174010c078c7585a74cd8cd0e2"
+ integrity sha512-+IF9hfUFOrYOOaKyfaI7h7dquUIOgyEMoQMLA7OP5FxegKA2+XdXThAZ9TU2kucfhDH7rfMHs1oPYziVGWRnZA==
dependencies:
- chokidar "^2.0.2"
+ chokidar "^2.1.8"
graceful-fs "^4.1.2"
neo-async "^2.5.0"
@@ -11105,7 +13017,7 @@
resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-4.0.2.tgz#a855980b1f0b6b359ba1d5d9fb39ae941faa63ad"
integrity sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==
-webpack-dev-middleware@^3.5.1:
+webpack-dev-middleware@^3.7.2:
version "3.7.2"
resolved "https://registry.yarnpkg.com/webpack-dev-middleware/-/webpack-dev-middleware-3.7.2.tgz#0019c3db716e3fa5cecbf64f2ab88a74bab331f3"
integrity sha512-1xC42LxbYoqLNAhV6YzTYacicgMZQTqRd27Sim9wn5hJrX3I5nxYy1SxSd4+gjUFsz1dQFj+yEe6zEVmSkeJjw==
@@ -11116,41 +13028,44 @@
range-parser "^1.2.1"
webpack-log "^2.0.0"
-webpack-dev-server@3.2.1:
- version "3.2.1"
- resolved "https://registry.yarnpkg.com/webpack-dev-server/-/webpack-dev-server-3.2.1.tgz#1b45ce3ecfc55b6ebe5e36dab2777c02bc508c4e"
- integrity sha512-sjuE4mnmx6JOh9kvSbPYw3u/6uxCLHNWfhWaIPwcXWsvWOPN+nc5baq4i9jui3oOBRXGonK9+OI0jVkaz6/rCw==
+webpack-dev-server@3.10.3:
+ version "3.10.3"
+ resolved "https://registry.yarnpkg.com/webpack-dev-server/-/webpack-dev-server-3.10.3.tgz#f35945036813e57ef582c2420ef7b470e14d3af0"
+ integrity sha512-e4nWev8YzEVNdOMcNzNeCN947sWJNd43E5XvsJzbAL08kGc2frm1tQ32hTJslRS+H65LCb/AaUCYU7fjHCpDeQ==
dependencies:
ansi-html "0.0.7"
bonjour "^3.5.0"
- chokidar "^2.0.0"
- compression "^1.5.2"
- connect-history-api-fallback "^1.3.0"
+ chokidar "^2.1.8"
+ compression "^1.7.4"
+ connect-history-api-fallback "^1.6.0"
debug "^4.1.1"
- del "^3.0.0"
- express "^4.16.2"
- html-entities "^1.2.0"
- http-proxy-middleware "^0.19.1"
+ del "^4.1.1"
+ express "^4.17.1"
+ html-entities "^1.2.1"
+ http-proxy-middleware "0.19.1"
import-local "^2.0.0"
- internal-ip "^4.2.0"
+ internal-ip "^4.3.0"
ip "^1.1.5"
- killable "^1.0.0"
- loglevel "^1.4.1"
- opn "^5.1.0"
- portfinder "^1.0.9"
+ is-absolute-url "^3.0.3"
+ killable "^1.0.1"
+ loglevel "^1.6.6"
+ opn "^5.5.0"
+ p-retry "^3.0.1"
+ portfinder "^1.0.25"
schema-utils "^1.0.0"
- selfsigned "^1.9.1"
- semver "^5.6.0"
- serve-index "^1.7.2"
+ selfsigned "^1.10.7"
+ semver "^6.3.0"
+ serve-index "^1.9.1"
sockjs "0.3.19"
- sockjs-client "1.3.0"
- spdy "^4.0.0"
- strip-ansi "^3.0.0"
+ sockjs-client "1.4.0"
+ spdy "^4.0.1"
+ strip-ansi "^3.0.1"
supports-color "^6.1.0"
url "^0.11.0"
- webpack-dev-middleware "^3.5.1"
+ webpack-dev-middleware "^3.7.2"
webpack-log "^2.0.0"
- yargs "12.0.2"
+ ws "^6.2.1"
+ yargs "12.0.5"
webpack-log@^2.0.0:
version "2.0.0"
@@ -11160,16 +13075,17 @@
ansi-colors "^3.0.0"
uuid "^3.3.2"
-webpack-manifest-plugin@2.0.4:
- version "2.0.4"
- resolved "https://registry.yarnpkg.com/webpack-manifest-plugin/-/webpack-manifest-plugin-2.0.4.tgz#e4ca2999b09557716b8ba4475fb79fab5986f0cd"
- integrity sha512-nejhOHexXDBKQOj/5v5IZSfCeTO3x1Dt1RZEcGfBSul891X/eLIcIVH31gwxPDdsi2Z8LKKFGpM4w9+oTBOSCg==
+webpack-manifest-plugin@2.2.0:
+ version "2.2.0"
+ resolved "https://registry.yarnpkg.com/webpack-manifest-plugin/-/webpack-manifest-plugin-2.2.0.tgz#19ca69b435b0baec7e29fbe90fb4015de2de4f16"
+ integrity sha512-9S6YyKKKh/Oz/eryM1RyLVDVmy3NSPV0JXMRhZ18fJsq+AwGxUY34X54VNwkzYcEmEkDwNxuEOboCZEebJXBAQ==
dependencies:
fs-extra "^7.0.0"
lodash ">=3.5 <5"
+ object.entries "^1.1.0"
tapable "^1.0.0"
-webpack-sources@^1.1.0, webpack-sources@^1.3.0, webpack-sources@^1.4.0:
+webpack-sources@^1.1.0, webpack-sources@^1.4.0, webpack-sources@^1.4.1, webpack-sources@^1.4.3:
version "1.4.3"
resolved "https://registry.yarnpkg.com/webpack-sources/-/webpack-sources-1.4.3.tgz#eedd8ec0b928fbf1cbfe994e22d2d890f330a933"
integrity sha512-lgTS3Xhv1lCOKo7SA5TjKXMjpSM4sBjNV5+q2bqesbSPs5FjGmU6jjtBSkX9b4qW87vDIsCIlUPOEhbZrMdjeQ==
@@ -11177,35 +13093,34 @@
source-list-map "^2.0.0"
source-map "~0.6.1"
-webpack@4.29.6:
- version "4.29.6"
- resolved "https://registry.yarnpkg.com/webpack/-/webpack-4.29.6.tgz#66bf0ec8beee4d469f8b598d3988ff9d8d90e955"
- integrity sha512-MwBwpiE1BQpMDkbnUUaW6K8RFZjljJHArC6tWQJoFm0oQtfoSebtg4Y7/QHnJ/SddtjYLHaKGX64CFjG5rehJw==
+webpack@4.42.0:
+ version "4.42.0"
+ resolved "https://registry.yarnpkg.com/webpack/-/webpack-4.42.0.tgz#b901635dd6179391d90740a63c93f76f39883eb8"
+ integrity sha512-EzJRHvwQyBiYrYqhyjW9AqM90dE4+s1/XtCfn7uWg6cS72zH+2VPFAlsnW0+W0cDi0XRjNKUMoJtpSi50+Ph6w==
dependencies:
"@webassemblyjs/ast" "1.8.5"
"@webassemblyjs/helper-module-context" "1.8.5"
"@webassemblyjs/wasm-edit" "1.8.5"
"@webassemblyjs/wasm-parser" "1.8.5"
- acorn "^6.0.5"
- acorn-dynamic-import "^4.0.0"
- ajv "^6.1.0"
- ajv-keywords "^3.1.0"
- chrome-trace-event "^1.0.0"
+ acorn "^6.2.1"
+ ajv "^6.10.2"
+ ajv-keywords "^3.4.1"
+ chrome-trace-event "^1.0.2"
enhanced-resolve "^4.1.0"
- eslint-scope "^4.0.0"
+ eslint-scope "^4.0.3"
json-parse-better-errors "^1.0.2"
- loader-runner "^2.3.0"
- loader-utils "^1.1.0"
- memory-fs "~0.4.1"
- micromatch "^3.1.8"
- mkdirp "~0.5.0"
- neo-async "^2.5.0"
- node-libs-browser "^2.0.0"
+ loader-runner "^2.4.0"
+ loader-utils "^1.2.3"
+ memory-fs "^0.4.1"
+ micromatch "^3.1.10"
+ mkdirp "^0.5.1"
+ neo-async "^2.6.1"
+ node-libs-browser "^2.2.1"
schema-utils "^1.0.0"
- tapable "^1.1.0"
- terser-webpack-plugin "^1.1.0"
- watchpack "^1.5.0"
- webpack-sources "^1.3.0"
+ tapable "^1.1.3"
+ terser-webpack-plugin "^1.4.3"
+ watchpack "^1.6.0"
+ webpack-sources "^1.4.1"
websocket-driver@>=0.5.1:
version "0.7.3"
@@ -11268,6 +13183,13 @@
dependencies:
isexe "^2.0.0"
+which@^2.0.1:
+ version "2.0.2"
+ resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1"
+ integrity sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==
+ dependencies:
+ isexe "^2.0.0"
+
widest-line@^2.0.0:
version "2.0.1"
resolved "https://registry.yarnpkg.com/widest-line/-/widest-line-2.0.1.tgz#7438764730ec7ef4381ce4df82fb98a53142a3fc"
@@ -11275,6 +13197,13 @@
dependencies:
string-width "^2.1.1"
+widest-line@^3.1.0:
+ version "3.1.0"
+ resolved "https://registry.yarnpkg.com/widest-line/-/widest-line-3.1.0.tgz#8292333bbf66cb45ff0de1603b136b7ae1496eca"
+ integrity sha512-NsmoXalsWVDMGupxZ5R08ka9flZjjiLvHVAWYOKtiKM8ujtZWr9cRffak+uSE48+Ob8ObalXpwyeUiyDD6QFgg==
+ dependencies:
+ string-width "^4.0.0"
+
word-wrap@~1.2.3:
version "1.2.3"
resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c"
@@ -11299,7 +13228,7 @@
dependencies:
workbox-core "^4.3.1"
-workbox-build@^4.2.0:
+workbox-build@^4.3.1:
version "4.3.1"
resolved "https://registry.yarnpkg.com/workbox-build/-/workbox-build-4.3.1.tgz#414f70fb4d6de47f6538608b80ec52412d233e64"
integrity sha512-UHdwrN3FrDvicM3AqJS/J07X0KXj67R8Cg0waq1MKEOqzo89ap6zh6LmaLnRAjpB+bDIz+7OlPye9iii9KBnxw==
@@ -11404,14 +13333,14 @@
resolved "https://registry.yarnpkg.com/workbox-sw/-/workbox-sw-4.3.1.tgz#df69e395c479ef4d14499372bcd84c0f5e246164"
integrity sha512-0jXdusCL2uC5gM3yYFT6QMBzKfBr2XTk0g5TPAV4y8IZDyVNDyj1a8uSXy3/XrvkVTmQvLN4O5k3JawGReXr9w==
-workbox-webpack-plugin@4.2.0:
- version "4.2.0"
- resolved "https://registry.yarnpkg.com/workbox-webpack-plugin/-/workbox-webpack-plugin-4.2.0.tgz#c94c3f69ff39c8a5b0c7e6bebc382cb53410a63d"
- integrity sha512-YZsiA+y/ns/GdWRaBsfYv8dln1ebWtGnJcTOg1ppO0pO1tScAHX0yGtHIjndxz3L/UUhE8b0NQE9KeLNwJwA5A==
+workbox-webpack-plugin@4.3.1:
+ version "4.3.1"
+ resolved "https://registry.yarnpkg.com/workbox-webpack-plugin/-/workbox-webpack-plugin-4.3.1.tgz#47ff5ea1cc074b6c40fb5a86108863a24120d4bd"
+ integrity sha512-gJ9jd8Mb8wHLbRz9ZvGN57IAmknOipD3W4XNE/Lk/4lqs5Htw4WOQgakQy/o/4CoXQlMCYldaqUg+EJ35l9MEQ==
dependencies:
"@babel/runtime" "^7.0.0"
json-stable-stringify "^1.0.1"
- workbox-build "^4.2.0"
+ workbox-build "^4.3.1"
workbox-window@^4.3.1:
version "4.3.1"
@@ -11420,7 +13349,7 @@
dependencies:
workbox-core "^4.3.1"
-worker-farm@^1.5.2, worker-farm@^1.7.0:
+worker-farm@^1.7.0:
version "1.7.0"
resolved "https://registry.yarnpkg.com/worker-farm/-/worker-farm-1.7.0.tgz#26a94c5391bbca926152002f69b84a4bf772e5a8"
integrity sha512-rvw3QTZc8lAxyVrqcSGVm5yP/IJ2UcB3U0graE3LCFoZ0Yn2x4EoVSqJKdB/T5M+FLcRPjz4TDacRf3OCfNUzw==
@@ -11474,6 +13403,16 @@
imurmurhash "^0.1.4"
signal-exit "^3.0.2"
+write-file-atomic@^3.0.0:
+ version "3.0.3"
+ resolved "https://registry.yarnpkg.com/write-file-atomic/-/write-file-atomic-3.0.3.tgz#56bd5c5a5c70481cd19c571bd39ab965a5de56e8"
+ integrity sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q==
+ dependencies:
+ imurmurhash "^0.1.4"
+ is-typedarray "^1.0.0"
+ signal-exit "^3.0.2"
+ typedarray-to-buffer "^3.1.5"
+
write@1.0.3:
version "1.0.3"
resolved "https://registry.yarnpkg.com/write/-/write-1.0.3.tgz#0800e14523b923a387e415123c865616aae0f5c3"
@@ -11488,7 +13427,7 @@
dependencies:
async-limiter "~1.0.0"
-ws@^6.1.2:
+ws@^6.1.2, ws@^6.2.1:
version "6.2.1"
resolved "https://registry.yarnpkg.com/ws/-/ws-6.2.1.tgz#442fdf0a47ed64f59b6a5d8ff130f4748ed524fb"
integrity sha512-GIyAXC2cB7LjvpgMt9EKS2ldqr0MTrORaleiOno6TweZ6r3TKtoFQWay/2PceJ3RuBasOHzXNn5Lrw1X0bEjqA==
@@ -11500,6 +13439,11 @@
resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-3.0.0.tgz#496b2cc109eca8dbacfe2dc72b603c17c5870ad4"
integrity sha1-SWsswQnsqNus/i3HK2A8F8WHCtQ=
+xdg-basedir@^4.0.0:
+ version "4.0.0"
+ resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-4.0.0.tgz#4bc8d9984403696225ef83a1573cbbcb4e79db13"
+ integrity sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q==
+
xml-name-validator@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a"
@@ -11510,10 +13454,61 @@
resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb"
integrity sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==
-xregexp@4.0.0:
- version "4.0.0"
- resolved "https://registry.yarnpkg.com/xregexp/-/xregexp-4.0.0.tgz#e698189de49dd2a18cc5687b05e17c8e43943020"
- integrity sha512-PHyM+sQouu7xspQQwELlGwwd05mXUFqwFYfqPO0cC7x4fxyHnnuetmQr6CjJiafIDoH4MogHb9dOoJzR/Y4rFg==
+xo@^0.30.0:
+ version "0.30.0"
+ resolved "https://registry.yarnpkg.com/xo/-/xo-0.30.0.tgz#e2991da22dea4c08c192cf74bba66cb634528ed2"
+ integrity sha512-755dmXTIj8uHv0lUZmDrtQb98wdu6ryqgcy8jvXBZgABz+kHm12IlzRQP+bO+w1u6NqkQB0Nn0Bh6Uv+KhdYiA==
+ dependencies:
+ "@typescript-eslint/eslint-plugin" "^2.29.0"
+ "@typescript-eslint/parser" "^2.29.0"
+ arrify "^2.0.1"
+ cosmiconfig "^6.0.0"
+ debug "^4.1.0"
+ eslint "^6.8.0"
+ eslint-config-prettier "^6.11.0"
+ eslint-config-xo "^0.29.0"
+ eslint-config-xo-typescript "^0.28.0"
+ eslint-formatter-pretty "^3.0.1"
+ eslint-import-resolver-webpack "^0.12.1"
+ eslint-plugin-ava "^10.0.1"
+ eslint-plugin-eslint-comments "^3.1.2"
+ eslint-plugin-import "^2.20.1"
+ eslint-plugin-no-use-extend-native "^0.5.0"
+ eslint-plugin-node "^11.0.0"
+ eslint-plugin-prettier "^3.1.3"
+ eslint-plugin-promise "^4.2.1"
+ eslint-plugin-unicorn "^19.0.0"
+ find-cache-dir "^3.3.1"
+ find-up "^4.1.0"
+ fs-extra "^9.0.0"
+ get-stdin "^7.0.0"
+ globby "^9.0.0"
+ has-flag "^4.0.0"
+ imurmurhash "^0.1.4"
+ is-path-inside "^3.0.2"
+ json-stable-stringify-without-jsonify "^1.0.1"
+ json5 "^2.1.1"
+ lodash "^4.17.15"
+ meow "^5.0.0"
+ micromatch "^4.0.2"
+ open-editor "^2.0.1"
+ p-reduce "^2.1.0"
+ path-exists "^4.0.0"
+ prettier "2.0.4"
+ resolve-cwd "^3.0.0"
+ resolve-from "^5.0.0"
+ semver "^7.3.2"
+ slash "^3.0.0"
+ to-absolute-glob "^2.0.2"
+ typescript "^3.0.0"
+ update-notifier "^4.0.0"
+
+xregexp@^4.3.0:
+ version "4.3.0"
+ resolved "https://registry.yarnpkg.com/xregexp/-/xregexp-4.3.0.tgz#7e92e73d9174a99a59743f67a4ce879a04b5ae50"
+ integrity sha512-7jXDIFXh5yJ/orPn4SXjuVrWWoi4Cr8jfV1eHv9CixKSbU+jY4mxfrBwAuDvupPNKpMUY+FeIqsVw/JLT9+B8g==
+ dependencies:
+ "@babel/runtime-corejs3" "^7.8.3"
xtend@^4.0.0, xtend@~4.0.1:
version "4.0.2"
@@ -11535,13 +13530,33 @@
resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd"
integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==
-yargs-parser@^10.1.0:
+yallist@^4.0.0:
+ version "4.0.0"
+ resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72"
+ integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==
+
+yaml@^1.7.2:
+ version "1.9.2"
+ resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.9.2.tgz#f0cfa865f003ab707663e4f04b3956957ea564ed"
+ integrity sha512-HPT7cGGI0DuRcsO51qC1j9O16Dh1mZ2bnXwsi0jrSpsLz0WxOLSLXfkABVl6bZO629py3CU+OMJtpNHDLB97kg==
+ dependencies:
+ "@babel/runtime" "^7.9.2"
+
+yargs-parser@^10.0.0:
version "10.1.0"
resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-10.1.0.tgz#7202265b89f7e9e9f2e5765e0fe735a905edbaa8"
integrity sha512-VCIyR1wJoEBZUqk5PA+oOBF6ypbwh5aNB3I50guxAL/quggdfs4TtNHQrSazFA3fYZ+tEqfs0zIGlv0c/rgjbQ==
dependencies:
camelcase "^4.1.0"
+yargs-parser@^11.1.1:
+ version "11.1.1"
+ resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-11.1.1.tgz#879a0865973bca9f6bab5cbdf3b1c67ec7d3bcf4"
+ integrity sha512-C6kB/WJDiaxONLJQnF8ccx9SEeoTTLek8RVbaOIsrAUS8VrBEXfmeSnCZxygc+XC2sNMBIwOOnfcxiynjHsVSQ==
+ dependencies:
+ camelcase "^5.0.0"
+ decamelize "^1.2.0"
+
yargs-parser@^13.1.1:
version "13.1.1"
resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-13.1.1.tgz#d26058532aa06d365fe091f6a1fc06b2f7e5eca0"
@@ -11558,13 +13573,13 @@
camelcase "^5.0.0"
decamelize "^1.2.0"
-yargs@12.0.2:
- version "12.0.2"
- resolved "https://registry.yarnpkg.com/yargs/-/yargs-12.0.2.tgz#fe58234369392af33ecbef53819171eff0f5aadc"
- integrity sha512-e7SkEx6N6SIZ5c5H22RTZae61qtn3PYUE8JYbBFlK9sYmh3DMQ6E5ygtaG/2BW0JZi4WGgTR2IV5ChqlqrDGVQ==
+yargs@12.0.5:
+ version "12.0.5"
+ resolved "https://registry.yarnpkg.com/yargs/-/yargs-12.0.5.tgz#05f5997b609647b64f66b81e3b4b10a368e7ad13"
+ integrity sha512-Lhz8TLaYnxq/2ObqHDql8dX8CJi97oHxrjUcYtzKbbykPtVW9WB+poxI+NM2UIzsMgNCZTIf0AQwsjK5yMAqZw==
dependencies:
cliui "^4.0.0"
- decamelize "^2.0.0"
+ decamelize "^1.2.0"
find-up "^3.0.0"
get-caller-file "^1.0.1"
os-locale "^3.0.0"
@@ -11574,7 +13589,7 @@
string-width "^2.0.0"
which-module "^2.0.0"
y18n "^3.2.1 || ^4.0.0"
- yargs-parser "^10.1.0"
+ yargs-parser "^11.1.1"
yargs@^13.3.0:
version "13.3.0"
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/AbstractOMMetadataManagerTest.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java
similarity index 79%
rename from hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/AbstractOMMetadataManagerTest.java
rename to hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java
index 08da2b3..848a2af 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/AbstractOMMetadataManagerTest.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java
@@ -44,24 +44,22 @@
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
import org.apache.hadoop.ozone.recon.recovery.ReconOmMetadataManagerImpl;
import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
-import org.junit.Rule;
-import org.junit.rules.TemporaryFolder;
/**
- * Utility methods for test classes.
+ * Utility methods for creating OM related metadata managers and objects.
*/
-public abstract class AbstractOMMetadataManagerTest {
+public final class OMMetadataManagerTestUtils {
- @Rule
- public TemporaryFolder temporaryFolder = new TemporaryFolder();
+ private OMMetadataManagerTestUtils() {
+ }
/**
* Create a new OM Metadata manager instance with default volume and bucket.
* @throws IOException ioEx
*/
- protected OMMetadataManager initializeNewOmMetadataManager()
+ public static OMMetadataManager initializeNewOmMetadataManager(
+ File omDbDir)
throws IOException {
- File omDbDir = temporaryFolder.newFolder();
OzoneConfiguration omConfiguration = new OzoneConfiguration();
omConfiguration.set(OZONE_OM_DB_DIRS,
omDbDir.getAbsolutePath());
@@ -94,9 +92,9 @@
* Create an empty OM Metadata manager instance.
* @throws IOException ioEx
*/
- protected OMMetadataManager initializeEmptyOmMetadataManager()
+ public static OMMetadataManager initializeEmptyOmMetadataManager(
+ File omDbDir)
throws IOException {
- File omDbDir = temporaryFolder.newFolder();
OzoneConfiguration omConfiguration = new OzoneConfiguration();
omConfiguration.set(OZONE_OM_DB_DIRS,
omDbDir.getAbsolutePath());
@@ -104,19 +102,19 @@
}
/**
- * Get an instance of Recon OM Metadata manager.
+ * Given an underlying OM DB, return an instance of Recon OM Metadata
+ * manager.
* @return ReconOMMetadataManager
* @throws IOException when creating the RocksDB instance.
*/
- protected ReconOMMetadataManager getTestMetadataManager(
- OMMetadataManager omMetadataManager)
+ public static ReconOMMetadataManager getTestReconOmMetadataManager(
+ OMMetadataManager omMetadataManager, File reconOmDbDir)
throws IOException {
DBCheckpoint checkpoint = omMetadataManager.getStore()
.getCheckpoint(true);
assertNotNull(checkpoint.getCheckpointLocation());
- File reconOmDbDir = temporaryFolder.newFolder();
OzoneConfiguration configuration = new OzoneConfiguration();
configuration.set(OZONE_RECON_OM_SNAPSHOT_DB_DIR, reconOmDbDir
.getAbsolutePath());
@@ -134,7 +132,7 @@
* Write a key to OM instance.
* @throws IOException while writing.
*/
- protected void writeDataToOm(OMMetadataManager omMetadataManager,
+ public static void writeDataToOm(OMMetadataManager omMetadataManager,
String key) throws IOException {
String omKey = omMetadataManager.getOzoneKey("sampleVol",
@@ -154,7 +152,7 @@
* Write a key to OM instance.
* @throws IOException while writing.
*/
- protected void writeDataToOm(OMMetadataManager omMetadataManager,
+ public static void writeDataToOm(OMMetadataManager omMetadataManager,
String key,
String bucket,
String volume,
@@ -177,34 +175,6 @@
}
/**
- * Write a key to OM instance.
- * @throws IOException while writing.
- */
- protected void writeDataToOm(OMMetadataManager omMetadataManager,
- String key,
- String bucket,
- String volume,
- Long dataSize,
- List<OmKeyLocationInfoGroup>
- omKeyLocationInfoGroupList)
- throws IOException {
-
- String omKey = omMetadataManager.getOzoneKey(volume,
- bucket, key);
-
- omMetadataManager.getKeyTable().put(omKey,
- new OmKeyInfo.Builder()
- .setBucketName(bucket)
- .setVolumeName(volume)
- .setKeyName(key)
- .setDataSize(dataSize)
- .setReplicationFactor(HddsProtos.ReplicationFactor.ONE)
- .setReplicationType(HddsProtos.ReplicationType.STAND_ALONE)
- .setOmKeyLocationInfos(omKeyLocationInfoGroupList)
- .build());
- }
-
- /**
* Return random pipeline.
* @return pipeline
*/
@@ -232,7 +202,7 @@
* @param pipeline pipeline
* @return new instance of OmKeyLocationInfo
*/
- protected OmKeyLocationInfo getOmKeyLocationInfo(BlockID blockID,
+ public static OmKeyLocationInfo getOmKeyLocationInfo(BlockID blockID,
Pipeline pipeline) {
return new OmKeyLocationInfo.Builder()
.setBlockID(blockID)
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/ReconTestInjector.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/ReconTestInjector.java
new file mode 100644
index 0000000..b852953
--- /dev/null
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/ReconTestInjector.java
@@ -0,0 +1,337 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon;
+
+import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_DATANODE_ADDRESS_KEY;
+import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_DB_DIR;
+import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_OM_SNAPSHOT_DB_DIR;
+import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SCM_DB_DIR;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.ozone.recon.persistence.AbstractReconSqlDBTest;
+import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
+import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider;
+import org.apache.hadoop.ozone.recon.spi.OzoneManagerServiceProvider;
+import org.apache.hadoop.ozone.recon.spi.impl.ContainerDBServiceProviderImpl;
+import org.apache.hadoop.ozone.recon.spi.impl.ReconContainerDBProvider;
+import org.junit.Assert;
+import org.junit.rules.TemporaryFolder;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+import com.google.inject.Singleton;
+
+/**
+ * Class to setup a recon test injector, with any combination of sub modules
+ * that are specified. This Recon specific abstraction to Guice API has
+ * been created to simplify the process of setting up a test environment for
+ * unit testing.
+ */
+public class ReconTestInjector {
+
+ private Injector injector;
+ private OzoneManagerServiceProvider ozoneManagerServiceProvider;
+ private ReconOMMetadataManager reconOMMetadataManager;
+ private OzoneStorageContainerManager reconScm;
+ private AbstractReconSqlDBTest reconSqlDB;
+ private boolean withContainerDB = false;
+ private List<Module> additionalModules = new ArrayList<>();
+ private boolean withReconSqlDb = false;
+ private TemporaryFolder temporaryFolder;
+ private Map<Class, Class> extraInheritedBindings = new HashMap<>();
+ private Map<Class, Object> extraInstanceBindings = new HashMap<>();
+ private Set<Class> extraClassBindings = new HashSet<>();
+
+ public ReconTestInjector(TemporaryFolder temporaryFolder) {
+ this.temporaryFolder = temporaryFolder;
+ }
+
+ public void setWithReconSqlDb(boolean withReconSqlDb) {
+ this.withReconSqlDb = withReconSqlDb;
+ }
+
+ public void setOzoneManagerServiceProvider(
+ OzoneManagerServiceProvider ozoneManagerServiceProvider) {
+ this.ozoneManagerServiceProvider = ozoneManagerServiceProvider;
+ }
+
+ public void setReconOMMetadataManager(
+ ReconOMMetadataManager reconOMMetadataManager) {
+ this.reconOMMetadataManager = reconOMMetadataManager;
+ }
+
+ public void setReconScm(OzoneStorageContainerManager reconScm) {
+ this.reconScm = reconScm;
+ }
+
+ public void withContainerDB(boolean containerDbIncluded) {
+ this.withContainerDB = containerDbIncluded;
+ }
+
+ public OzoneManagerServiceProvider getOzoneManagerServiceProvider() {
+ return ozoneManagerServiceProvider;
+ }
+
+ public ReconOMMetadataManager getReconOMMetadataManager() {
+ return reconOMMetadataManager;
+ }
+
+ public OzoneStorageContainerManager getReconScm() {
+ return reconScm;
+ }
+
+ public List<Module> getAdditionalModules() {
+ return additionalModules;
+ }
+
+ public Map<Class, Object> getExtraInstanceBindings() {
+ return extraInstanceBindings;
+ }
+
+ public Map<Class, Class> getExtraInheritedBindings() {
+ return extraInheritedBindings;
+ }
+
+ public Set<Class> getExtraClassBindings() {
+ return extraClassBindings;
+ }
+
+ /**
+ * Wrapper to get the bound instance.
+ * @param type type
+ * @param <T> type
+ * @return bound instance of type T.
+ */
+ public <T> T getInstance(Class<T> type) {
+ return injector.getInstance(type);
+ }
+
+ /**
+ * The goal of the class is to discourage the use of injector to
+ * create more child injectors explicitly.
+ * Use this API wisely!
+ * @return injector.
+ */
+ public Injector getInjector() {
+ return injector;
+ }
+
+ void setupInjector() throws IOException {
+ List<Module> modules = new ArrayList<>();
+
+ modules.add(new AbstractModule() {
+ @Override
+ protected void configure() {
+ try {
+ bind(OzoneConfiguration.class).toInstance(
+ getTestOzoneConfiguration(temporaryFolder.newFolder()));
+
+ if (reconOMMetadataManager != null) {
+ bind(ReconOMMetadataManager.class)
+ .toInstance(reconOMMetadataManager);
+ }
+
+ if (ozoneManagerServiceProvider != null) {
+ bind(OzoneManagerServiceProvider.class)
+ .toInstance(ozoneManagerServiceProvider);
+ }
+
+ if (reconScm != null) {
+ bind(OzoneStorageContainerManager.class).toInstance(reconScm);
+ }
+
+ if (withContainerDB) {
+ bind(ContainerDBServiceProvider.class)
+ .to(ContainerDBServiceProviderImpl.class).in(Singleton.class);
+ bind(DBStore.class).toProvider(ReconContainerDBProvider.class).
+ in(Singleton.class);
+ }
+
+ for (Map.Entry<Class, Object> entry :
+ extraInstanceBindings.entrySet()) {
+ bind(entry.getKey()).toInstance(entry.getValue());
+ }
+
+ for (Map.Entry<Class, Class> entry :
+ extraInheritedBindings.entrySet()) {
+ bind(entry.getKey()).to(entry.getValue()).in(Singleton.class);
+ }
+
+ for (Class type : extraClassBindings) {
+ bind(type).in(Singleton.class);
+ }
+ } catch (IOException e) {
+ Assert.fail();
+ }
+ }
+ });
+
+ if (CollectionUtils.isNotEmpty(additionalModules)) {
+ modules.addAll(additionalModules);
+ }
+
+ if (withReconSqlDb) {
+ reconSqlDB = new AbstractReconSqlDBTest();
+ modules.addAll(reconSqlDB.getReconSqlDBModules());
+ }
+
+ injector = Guice.createInjector(modules);
+ if (reconSqlDB != null) {
+ reconSqlDB.createSchema(injector);
+ }
+ }
+
+ public OzoneConfiguration getTestOzoneConfiguration(
+ File dir) {
+ OzoneConfiguration configuration = new OzoneConfiguration();
+ configuration.set(OZONE_RECON_OM_SNAPSHOT_DB_DIR, dir.getAbsolutePath());
+ configuration.set(OZONE_RECON_DB_DIR, dir.getAbsolutePath());
+ configuration.set(OZONE_RECON_SCM_DB_DIR, dir.getAbsolutePath());
+ configuration.set(OZONE_RECON_DATANODE_ADDRESS_KEY,
+ "0.0.0.0:0");
+ return configuration;
+ }
+
+
+ /**
+ * Builder for Recon Test Injector.
+ */
+ public static class Builder {
+ private ReconTestInjector reconTestInjector;
+
+ public Builder(TemporaryFolder temporaryFolder) {
+ reconTestInjector = new ReconTestInjector(temporaryFolder);
+ }
+
+ /**
+ * Use if you need the Recon SQL DB instance.
+ */
+ public Builder withReconSqlDb() {
+ reconTestInjector.setWithReconSqlDb(true);
+ return this;
+ }
+
+ /**
+ * Pass in your Ozone manager service provider implementation, maybe with
+ * mocked behavior.
+ * @param ozoneManagerServiceProvider instance
+ */
+ public Builder withOmServiceProvider(
+ OzoneManagerServiceProvider ozoneManagerServiceProvider) {
+ reconTestInjector.setOzoneManagerServiceProvider(
+ ozoneManagerServiceProvider);
+ return this;
+ }
+
+ /**
+ * Pass in your ReconOMMetadataManager implementation, maybe with
+ * mocked behavior.
+ * @param reconOm instance
+ */
+ public Builder withReconOm(ReconOMMetadataManager reconOm) {
+ reconTestInjector.setReconOMMetadataManager(reconOm);
+ return this;
+ }
+
+ /**
+ * Pass in your Recon SCM implementation.
+ * @param reconScm instance
+ * @return Builder.
+ */
+ public Builder withReconScm(OzoneStorageContainerManager reconScm) {
+ reconTestInjector.setReconScm(reconScm);
+ return this;
+ }
+
+ /**
+ * Use if you need the ReconContainerDB. Bound to default implementation.
+ * @return Builder.
+ */
+ public Builder withContainerDB() {
+ reconTestInjector.withContainerDB(true);
+ return this;
+ }
+
+ /**
+ * Add binding of the type bind(A.class).toInstance(AImpl).
+ * @param type class type
+ * @param instance instance
+ * @return Builder.
+ */
+ public Builder addBinding(Class type, Object instance) {
+ reconTestInjector.getExtraInstanceBindings().put(type, instance);
+ return this;
+ }
+
+ /**
+ * Add binding of the type bind(A.class).
+ * @param type class type
+ * @return Builder.
+ */
+ public Builder addBinding(Class type) {
+ reconTestInjector.getExtraClassBindings().add(type);
+ return this;
+ }
+
+ /**
+ * Add binding of the type bind(A.class).to(B.class) where B extends A.
+ * @param type class type
+ * @return Builder.
+ */
+ public Builder addBinding(Class type, Class inheritedType) {
+ reconTestInjector.getExtraInheritedBindings().put(type, inheritedType);
+ return this;
+ }
+
+ /**
+ * If you really need to pass in more injector modules for extending the
+ * set of classes bound, use this.
+ * @param module external module.
+ * @return Builder.
+ */
+ public Builder addModule(Module module) {
+ reconTestInjector.getAdditionalModules().add(module);
+ return this;
+ }
+
+ /**
+ * Build the whole graph of classes.
+ * @return ReconTestInjector
+ * @throws IOException on error.
+ */
+ public ReconTestInjector build() throws IOException {
+ reconTestInjector.setupInjector();
+ return reconTestInjector;
+ }
+ }
+}
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconUtils.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconUtils.java
index 6bb8993..0951299 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconUtils.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconUtils.java
@@ -22,6 +22,7 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -34,20 +35,17 @@
import java.io.InputStream;
import java.nio.charset.Charset;
import java.nio.file.Paths;
+import java.net.URLConnection;
+import java.net.URL;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.http.HttpEntity;
-import org.apache.http.StatusLine;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.hadoop.hdfs.web.URLConnectionFactory;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
-
/**
* Test Recon Utility methods.
*/
@@ -137,20 +135,7 @@
@Test
public void testMakeHttpCall() throws Exception {
-
- CloseableHttpClient httpClientMock = mock(CloseableHttpClient.class);
String url = "http://localhost:9874/dbCheckpoint";
-
- CloseableHttpResponse httpResponseMock = mock(CloseableHttpResponse.class);
- when(httpClientMock.execute(any(HttpGet.class)))
- .thenReturn(httpResponseMock);
-
- StatusLine statusLineMock = mock(StatusLine.class);
- when(statusLineMock.getStatusCode()).thenReturn(200);
- when(httpResponseMock.getStatusLine()).thenReturn(statusLineMock);
-
- HttpEntity httpEntityMock = mock(HttpEntity.class);
- when(httpResponseMock.getEntity()).thenReturn(httpEntityMock);
File file1 = Paths.get(folder.getRoot().getPath(), "file1")
.toFile();
BufferedWriter writer = new BufferedWriter(new FileWriter(
@@ -159,16 +144,17 @@
writer.close();
InputStream fileInputStream = new FileInputStream(file1);
- when(httpEntityMock.getContent()).thenReturn(new InputStream() {
- @Override
- public int read() throws IOException {
- return fileInputStream.read();
- }
- });
-
- InputStream inputStream = new ReconUtils()
- .makeHttpCall(httpClientMock, url);
- String contents = IOUtils.toString(inputStream, Charset.defaultCharset());
+ String contents;
+ URLConnectionFactory connectionFactoryMock =
+ mock(URLConnectionFactory.class);
+ URLConnection urlConnectionMock = mock(URLConnection.class);
+ when(urlConnectionMock.getInputStream()).thenReturn(fileInputStream);
+ when(connectionFactoryMock.openConnection(any(URL.class), anyBoolean()))
+ .thenReturn(urlConnectionMock);
+ try (InputStream inputStream = new ReconUtils()
+ .makeHttpCall(connectionFactoryMock, url, false)) {
+ contents = IOUtils.toString(inputStream, Charset.defaultCharset());
+ }
assertEquals("File 1 Contents", contents);
}
@@ -204,4 +190,4 @@
File latestValidFile = reconUtils.getLastKnownDB(newDir, "valid");
assertTrue(latestValidFile.getName().equals("valid_2"));
}
-}
\ No newline at end of file
+}
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
index 5b373cc..c59c237 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
@@ -18,21 +18,30 @@
package org.apache.hadoop.ozone.recon.api;
+import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getOmKeyLocationInfo;
+import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandomPipeline;
+import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager;
+import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.initializeNewOmMetadataManager;
+import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDataToOm;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import javax.ws.rs.core.Response;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.ContainerManager;
@@ -42,14 +51,14 @@
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
-import org.apache.hadoop.ozone.recon.AbstractOMMetadataManagerTest;
-import org.apache.hadoop.ozone.recon.GuiceInjectorUtilsForTestsImpl;
+import org.apache.hadoop.ozone.recon.ReconTestInjector;
import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
import org.apache.hadoop.ozone.recon.api.types.ContainersResponse;
import org.apache.hadoop.ozone.recon.api.types.KeyMetadata;
import org.apache.hadoop.ozone.recon.api.types.KeysResponse;
import org.apache.hadoop.ozone.recon.api.types.MissingContainerMetadata;
import org.apache.hadoop.ozone.recon.api.types.MissingContainersResponse;
+import org.apache.hadoop.ozone.recon.persistence.ContainerSchemaManager;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
import org.apache.hadoop.ozone.recon.scm.ReconContainerManager;
import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade;
@@ -59,43 +68,34 @@
import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl;
import org.apache.hadoop.ozone.recon.tasks.ContainerKeyMapperTask;
import org.apache.hadoop.hdds.utils.db.Table;
-import org.hadoop.ozone.recon.schema.ReconTaskSchemaDefinition;
-import org.hadoop.ozone.recon.schema.StatsSchemaDefinition;
-import org.hadoop.ozone.recon.schema.UtilizationSchemaDefinition;
-import org.hadoop.ozone.recon.schema.tables.daos.MissingContainersDao;
-import org.hadoop.ozone.recon.schema.tables.daos.ReconTaskStatusDao;
-import org.hadoop.ozone.recon.schema.tables.pojos.MissingContainers;
-import org.jooq.Configuration;
+import org.hadoop.ozone.recon.schema.tables.pojos.ContainerHistory;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
-
-import com.google.inject.AbstractModule;
-import com.google.inject.Injector;
+import org.junit.rules.TemporaryFolder;
/**
* Test for container endpoint.
*/
-public class TestContainerEndpoint extends AbstractOMMetadataManagerTest {
+public class TestContainerEndpoint {
+
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
private ContainerDBServiceProvider containerDbServiceProvider;
private ContainerEndpoint containerEndpoint;
- private GuiceInjectorUtilsForTestsImpl guiceInjectorTest =
- new GuiceInjectorUtilsForTestsImpl();
private boolean isSetupDone = false;
+ private ContainerSchemaManager containerSchemaManager;
private ReconOMMetadataManager reconOMMetadataManager;
- private MissingContainersDao missingContainersDao;
private ContainerID containerID = new ContainerID(1L);
private PipelineID pipelineID;
private long keyCount = 5L;
- private void initializeInjector() throws Exception {
- reconOMMetadataManager = getTestMetadataManager(
- initializeNewOmMetadataManager());
- OzoneManagerServiceProviderImpl ozoneManagerServiceProvider =
- mock(OzoneManagerServiceProviderImpl.class);
- Injector parentInjector = guiceInjectorTest.getInjector(
- ozoneManagerServiceProvider, reconOMMetadataManager, temporaryFolder);
+ private void initializeInjector() throws Exception {
+ reconOMMetadataManager = getTestReconOmMetadataManager(
+ initializeNewOmMetadataManager(temporaryFolder.newFolder()),
+ temporaryFolder.newFolder());
Pipeline pipeline = getRandomPipeline();
pipelineID = pipeline.getId();
@@ -110,51 +110,31 @@
new ContainerInfo.Builder()
.setContainerID(containerID.getId())
.setNumberOfKeys(keyCount)
+ .setReplicationFactor(ReplicationFactor.THREE)
.setPipelineID(pipelineID)
.build()
);
when(mockReconSCM.getContainerManager())
.thenReturn(mockContainerManager);
- Injector injector = parentInjector.createChildInjector(
- new AbstractModule() {
- @Override
- protected void configure() {
- Configuration sqlConfiguration =
- parentInjector.getInstance((Configuration.class));
+ ReconTestInjector reconTestInjector =
+ new ReconTestInjector.Builder(temporaryFolder)
+ .withReconSqlDb()
+ .withReconOm(reconOMMetadataManager)
+ .withOmServiceProvider(mock(OzoneManagerServiceProviderImpl.class))
+ .withReconScm(mockReconSCM)
+ .withContainerDB()
+ .addBinding(StorageContainerServiceProvider.class,
+ mock(StorageContainerServiceProviderImpl.class))
+ .addBinding(ContainerEndpoint.class)
+ .addBinding(ContainerSchemaManager.class)
+ .build();
- try {
- ReconTaskSchemaDefinition taskSchemaDefinition = parentInjector
- .getInstance(ReconTaskSchemaDefinition.class);
- taskSchemaDefinition.initializeSchema();
- } catch (Exception e) {
- Assert.fail(e.getMessage());
- }
-
- ReconTaskStatusDao reconTaskStatusDao =
- new ReconTaskStatusDao(sqlConfiguration);
-
- bind(ReconTaskStatusDao.class).toInstance(reconTaskStatusDao);
-
- StorageContainerServiceProvider mockScmServiceProvider = mock(
- StorageContainerServiceProviderImpl.class);
- bind(StorageContainerServiceProvider.class)
- .toInstance(mockScmServiceProvider);
- bind(OzoneStorageContainerManager.class)
- .toInstance(mockReconSCM);
- bind(ContainerEndpoint.class);
- }
- });
- containerEndpoint = injector.getInstance(ContainerEndpoint.class);
- containerDbServiceProvider = injector.getInstance(
- ContainerDBServiceProvider.class);
- StatsSchemaDefinition schemaDefinition = injector.getInstance(
- StatsSchemaDefinition.class);
- schemaDefinition.initializeSchema();
- UtilizationSchemaDefinition utilizationSchemaDefinition =
- injector.getInstance(UtilizationSchemaDefinition.class);
- utilizationSchemaDefinition.initializeSchema();
- missingContainersDao = injector.getInstance(MissingContainersDao.class);
+ containerDbServiceProvider =
+ reconTestInjector.getInstance(ContainerDBServiceProvider.class);
+ containerEndpoint = reconTestInjector.getInstance(ContainerEndpoint.class);
+ containerSchemaManager =
+ reconTestInjector.getInstance(ContainerSchemaManager.class);
}
@Before
@@ -164,7 +144,6 @@
initializeInjector();
isSetupDone = true;
}
-
//Write Data to OM
Pipeline pipeline = getRandomPipeline();
@@ -244,8 +223,7 @@
Response response = containerEndpoint.getKeysForContainer(1L, -1, "");
- KeysResponse responseObject = (KeysResponse) response.getEntity();
- KeysResponse.KeysResponseData data = responseObject.getKeysResponseData();
+ KeysResponse data = (KeysResponse) response.getEntity();
Collection<KeyMetadata> keyMetadataList = data.getKeys();
assertEquals(3, data.getTotalCount());
@@ -272,16 +250,14 @@
assertEquals(104, blockIds.get(1L).iterator().next().getLocalID());
response = containerEndpoint.getKeysForContainer(3L, -1, "");
- responseObject = (KeysResponse) response.getEntity();
- data = responseObject.getKeysResponseData();
+ data = (KeysResponse) response.getEntity();
keyMetadataList = data.getKeys();
assertTrue(keyMetadataList.isEmpty());
assertEquals(0, data.getTotalCount());
// test if limit works as expected
response = containerEndpoint.getKeysForContainer(1L, 1, "");
- responseObject = (KeysResponse) response.getEntity();
- data = responseObject.getKeysResponseData();
+ data = (KeysResponse) response.getEntity();
keyMetadataList = data.getKeys();
assertEquals(1, keyMetadataList.size());
assertEquals(3, data.getTotalCount());
@@ -293,11 +269,9 @@
Response response = containerEndpoint.getKeysForContainer(
1L, -1, "/sampleVol/bucketOne/key_one");
- KeysResponse responseObject =
+ KeysResponse data =
(KeysResponse) response.getEntity();
- KeysResponse.KeysResponseData data =
- responseObject.getKeysResponseData();
assertEquals(3, data.getTotalCount());
Collection<KeyMetadata> keyMetadataList = data.getKeys();
@@ -312,8 +286,7 @@
response = containerEndpoint.getKeysForContainer(
1L, -1, StringUtils.EMPTY);
- responseObject = (KeysResponse) response.getEntity();
- data = responseObject.getKeysResponseData();
+ data = (KeysResponse) response.getEntity();
keyMetadataList = data.getKeys();
assertEquals(3, data.getTotalCount());
@@ -325,16 +298,14 @@
// test for negative cases
response = containerEndpoint.getKeysForContainer(
1L, -1, "/sampleVol/bucketOne/invalid_key");
- responseObject = (KeysResponse) response.getEntity();
- data = responseObject.getKeysResponseData();
+ data = (KeysResponse) response.getEntity();
keyMetadataList = data.getKeys();
assertEquals(3, data.getTotalCount());
assertEquals(0, keyMetadataList.size());
response = containerEndpoint.getKeysForContainer(
5L, -1, "");
- responseObject = (KeysResponse) response.getEntity();
- data = responseObject.getKeysResponseData();
+ data = (KeysResponse) response.getEntity();
keyMetadataList = data.getKeys();
assertEquals(0, keyMetadataList.size());
assertEquals(0, data.getTotalCount());
@@ -434,9 +405,13 @@
// Add missing containers to the database
long missingSince = System.currentTimeMillis();
- MissingContainers newRecord =
- new MissingContainers(1L, missingSince);
- missingContainersDao.insert(newRecord);
+ containerSchemaManager.addMissingContainer(1L, missingSince);
+
+ // Add container history for id 1
+ containerSchemaManager.upsertContainerHistory(1L, "host1", 1L);
+ containerSchemaManager.upsertContainerHistory(1L, "host2", 2L);
+ containerSchemaManager.upsertContainerHistory(1L, "host3", 3L);
+ containerSchemaManager.upsertContainerHistory(1L, "host4", 4L);
response = containerEndpoint.getMissingContainers();
responseObject = (MissingContainersResponse) response.getEntity();
@@ -448,7 +423,38 @@
assertEquals(containerID.getId(), container.getContainerID());
assertEquals(keyCount, container.getKeys());
assertEquals(pipelineID.getId(), container.getPipelineID());
- assertEquals(0, container.getDatanodes().size());
+ assertEquals(3, container.getReplicas().size());
assertEquals(missingSince, container.getMissingSince());
+
+ Set<String> datanodes = Collections.unmodifiableSet(
+ new HashSet<>(Arrays.asList("host2", "host3", "host4")));
+ List<ContainerHistory> containerReplicas = container.getReplicas();
+ containerReplicas.forEach(history -> {
+ Assert.assertTrue(datanodes.contains(history.getDatanodeHost()));
+ });
+ }
+
+ @Test
+ public void testGetReplicaHistoryForContainer() {
+ // Add container history for id 1
+ containerSchemaManager.upsertContainerHistory(1L, "host1", 1L);
+ containerSchemaManager.upsertContainerHistory(1L, "host2", 2L);
+ containerSchemaManager.upsertContainerHistory(1L, "host3", 3L);
+ containerSchemaManager.upsertContainerHistory(1L, "host4", 4L);
+ containerSchemaManager.upsertContainerHistory(1L, "host1", 5L);
+
+ Response response = containerEndpoint.getReplicaHistoryForContainer(1L);
+ List<ContainerHistory> histories =
+ (List<ContainerHistory>) response.getEntity();
+ Set<String> datanodes = Collections.unmodifiableSet(
+ new HashSet<>(Arrays.asList("host1", "host2", "host3", "host4")));
+ Assert.assertEquals(4, histories.size());
+ histories.forEach(history -> {
+ Assert.assertTrue(datanodes.contains(history.getDatanodeHost()));
+ if (history.getDatanodeHost().equals("host1")) {
+ Assert.assertEquals(1L, (long) history.getFirstReportTimestamp());
+ Assert.assertEquals(5L, (long) history.getLastReportTimestamp());
+ }
+ });
}
}
\ No newline at end of file
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java
index 6d5ea50..9234131 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java
@@ -18,9 +18,6 @@
package org.apache.hadoop.ozone.recon.api;
-import com.google.inject.AbstractModule;
-import com.google.inject.Injector;
-import com.google.inject.Singleton;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
@@ -42,31 +39,36 @@
import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
-import org.apache.hadoop.ozone.recon.AbstractOMMetadataManagerTest;
-import org.apache.hadoop.ozone.recon.GuiceInjectorUtilsForTestsImpl;
+import org.apache.hadoop.ozone.recon.ReconTestInjector;
import org.apache.hadoop.ozone.recon.api.types.ClusterStateResponse;
import org.apache.hadoop.ozone.recon.api.types.DatanodeMetadata;
import org.apache.hadoop.ozone.recon.api.types.DatanodesResponse;
import org.apache.hadoop.ozone.recon.api.types.PipelineMetadata;
import org.apache.hadoop.ozone.recon.api.types.PipelinesResponse;
+import org.apache.hadoop.ozone.recon.persistence.AbstractReconSqlDBTest;
+import org.apache.hadoop.ozone.recon.persistence.ContainerSchemaManager;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade;
import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider;
import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl;
import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl;
import org.apache.hadoop.test.LambdaTestUtils;
-import org.hadoop.ozone.recon.schema.ReconTaskSchemaDefinition;
-import org.hadoop.ozone.recon.schema.tables.daos.ReconTaskStatusDao;
-import org.jooq.Configuration;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails;
+import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandomPipeline;
+import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager;
+import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.initializeNewOmMetadataManager;
+import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDataToOm;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import javax.ws.rs.core.Response;
+
import java.io.IOException;
import java.util.UUID;
import java.util.concurrent.Callable;
@@ -74,7 +76,7 @@
/**
* Test for Recon API endpoints.
*/
-public class TestEndpoints extends AbstractOMMetadataManagerTest {
+public class TestEndpoints extends AbstractReconSqlDBTest {
private NodeEndpoint nodeEndpoint;
private PipelineEndpoint pipelineEndpoint;
private ClusterStateEndpoint clusterStateEndpoint;
@@ -84,85 +86,76 @@
private String pipelineId;
private DatanodeDetails datanodeDetails;
private DatanodeDetails datanodeDetails2;
- private GuiceInjectorUtilsForTestsImpl guiceInjectorTest =
- new GuiceInjectorUtilsForTestsImpl();
private long containerId = 1L;
private ContainerReportsProto containerReportsProto;
private DatanodeDetailsProto datanodeDetailsProto;
private Pipeline pipeline;
+ private final String host1 = "host1.datanode";
+ private final String host2 = "host2.datanode";
+ private final String ip1 = "1.1.1.1";
+ private final String ip2 = "2.2.2.2";
+
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
private void initializeInjector() throws IOException {
- reconOMMetadataManager = getTestMetadataManager(
- initializeNewOmMetadataManager());
- OzoneManagerServiceProviderImpl omServiceProviderMock =
- mock(OzoneManagerServiceProviderImpl.class);
- Injector parentInjector = guiceInjectorTest.getInjector(
- omServiceProviderMock, reconOMMetadataManager, temporaryFolder);
- Injector injector = parentInjector.createChildInjector(
- new AbstractModule() {
- @Override
- protected void configure() {
- try {
- datanodeDetails = randomDatanodeDetails();
- datanodeDetails2 = randomDatanodeDetails();
- pipeline = getRandomPipeline(datanodeDetails);
- pipelineId = pipeline.getId().getId().toString();
+ reconOMMetadataManager = getTestReconOmMetadataManager(
+ initializeNewOmMetadataManager(temporaryFolder.newFolder()),
+ temporaryFolder.newFolder());
+ datanodeDetails = randomDatanodeDetails();
+ datanodeDetails2 = randomDatanodeDetails();
+ datanodeDetails.setHostName(host1);
+ datanodeDetails.setIpAddress(ip1);
+ datanodeDetails2.setHostName(host2);
+ datanodeDetails2.setIpAddress(ip2);
+ pipeline = getRandomPipeline(datanodeDetails);
+ pipelineId = pipeline.getId().getId().toString();
- Configuration sqlConfiguration =
- parentInjector.getInstance((Configuration.class));
+ ContainerInfo containerInfo = new ContainerInfo.Builder()
+ .setContainerID(containerId)
+ .setReplicationFactor(ReplicationFactor.ONE)
+ .setState(LifeCycleState.OPEN)
+ .setOwner("test")
+ .setPipelineID(pipeline.getId())
+ .setReplicationType(ReplicationType.RATIS)
+ .build();
+ ContainerWithPipeline containerWithPipeline =
+ new ContainerWithPipeline(containerInfo, pipeline);
- ContainerInfo containerInfo = new ContainerInfo.Builder()
- .setContainerID(containerId)
- .setReplicationFactor(ReplicationFactor.ONE)
- .setState(LifeCycleState.OPEN)
- .setOwner("test")
- .setPipelineID(pipeline.getId())
- .setReplicationType(ReplicationType.RATIS)
- .build();
- ContainerWithPipeline containerWithPipeline =
- new ContainerWithPipeline(containerInfo, pipeline);
+ StorageContainerLocationProtocol mockScmClient = mock(
+ StorageContainerLocationProtocol.class);
+ StorageContainerServiceProvider mockScmServiceProvider = mock(
+ StorageContainerServiceProviderImpl.class);
+ when(mockScmServiceProvider.getPipeline(
+ pipeline.getId().getProtobuf())).thenReturn(pipeline);
+ when(mockScmServiceProvider.getContainerWithPipeline(containerId))
+ .thenReturn(containerWithPipeline);
- ReconTaskSchemaDefinition taskSchemaDefinition = parentInjector
- .getInstance(ReconTaskSchemaDefinition.class);
- taskSchemaDefinition.initializeSchema();
+ ReconTestInjector reconTestInjector =
+ new ReconTestInjector.Builder(temporaryFolder)
+ .withReconSqlDb()
+ .withReconOm(reconOMMetadataManager)
+ .withOmServiceProvider(mock(OzoneManagerServiceProviderImpl.class))
+ .addBinding(StorageContainerServiceProvider.class,
+ mockScmServiceProvider)
+ .addBinding(OzoneStorageContainerManager.class,
+ ReconStorageContainerManagerFacade.class)
+ .addBinding(ClusterStateEndpoint.class)
+ .addBinding(NodeEndpoint.class)
+ .addBinding(ContainerSchemaManager.class)
+ .addBinding(StorageContainerLocationProtocol.class, mockScmClient)
+ .build();
- ReconTaskStatusDao reconTaskStatusDao =
- new ReconTaskStatusDao(sqlConfiguration);
-
- bind(ReconTaskStatusDao.class).toInstance(reconTaskStatusDao);
-
- StorageContainerLocationProtocol mockScmClient = mock(
- StorageContainerLocationProtocol.class);
- StorageContainerServiceProvider mockScmServiceProvider = mock(
- StorageContainerServiceProviderImpl.class);
- when(mockScmServiceProvider.getPipeline(
- pipeline.getId().getProtobuf())).thenReturn(pipeline);
- when(mockScmServiceProvider.getContainerWithPipeline(containerId))
- .thenReturn(containerWithPipeline);
-
- bind(StorageContainerLocationProtocol.class)
- .toInstance(mockScmClient);
- bind(StorageContainerServiceProvider.class)
- .toInstance(mockScmServiceProvider);
- bind(OzoneStorageContainerManager.class)
- .to(ReconStorageContainerManagerFacade.class)
- .in(Singleton.class);
- bind(NodeEndpoint.class);
- } catch (Exception e) {
- Assert.fail(e.getMessage());
- }
- }
- });
-
- nodeEndpoint = injector.getInstance(NodeEndpoint.class);
- pipelineEndpoint = injector.getInstance(PipelineEndpoint.class);
- clusterStateEndpoint = injector.getInstance(ClusterStateEndpoint.class);
+ nodeEndpoint = reconTestInjector.getInstance(NodeEndpoint.class);
+ pipelineEndpoint = reconTestInjector.getInstance(PipelineEndpoint.class);
+ clusterStateEndpoint =
+ reconTestInjector.getInstance(ClusterStateEndpoint.class);
reconScm = (ReconStorageContainerManagerFacade)
- injector.getInstance(OzoneStorageContainerManager.class);
+ reconTestInjector.getInstance(OzoneStorageContainerManager.class);
}
@Before
- public void setUp() throws IOException {
+ public void setUp() throws Exception {
// The following setup runs only once
if (!isSetupDone) {
initializeInjector();
@@ -190,9 +183,9 @@
.addPipelineReport(pipelineReport).build();
datanodeDetailsProto =
DatanodeDetailsProto.newBuilder()
- .setHostName("host1.datanode")
+ .setHostName(host1)
.setUuid(datanodeId)
- .setIpAddress("1.1.1.1")
+ .setIpAddress(ip1)
.build();
StorageReportProto storageReportProto1 =
StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK)
@@ -213,9 +206,9 @@
DatanodeDetailsProto datanodeDetailsProto2 =
DatanodeDetailsProto.newBuilder()
- .setHostName("host2.datanode")
+ .setHostName(host2)
.setUuid(datanodeId2)
- .setIpAddress("2.2.2.2")
+ .setIpAddress(ip2)
.build();
StorageReportProto storageReportProto3 =
StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK)
@@ -282,10 +275,11 @@
writeDataToOm(reconOMMetadataManager, "key_three");
}
- private void testDatanodeResponse(DatanodeMetadata datanodeMetadata) {
+ private void testDatanodeResponse(DatanodeMetadata datanodeMetadata)
+ throws IOException {
String hostname = datanodeMetadata.getHostname();
switch (hostname) {
- case "host1.datanode":
+ case host1:
Assert.assertEquals(75000,
datanodeMetadata.getDatanodeStorageReport().getCapacity());
Assert.assertEquals(15400,
@@ -300,8 +294,11 @@
datanodeMetadata.getPipelines().get(0).getReplicationFactor());
Assert.assertEquals(pipeline.getType().toString(),
datanodeMetadata.getPipelines().get(0).getReplicationType());
+ Assert.assertEquals(pipeline.getLeaderNode().getHostName(),
+ datanodeMetadata.getPipelines().get(0).getLeaderNode());
+ Assert.assertEquals(1, datanodeMetadata.getLeaderCount());
break;
- case "host2.datanode":
+ case host2:
Assert.assertEquals(130000,
datanodeMetadata.getDatanodeStorageReport().getCapacity());
Assert.assertEquals(17800,
@@ -310,6 +307,7 @@
datanodeMetadata.getDatanodeStorageReport().getUsed());
Assert.assertEquals(0, datanodeMetadata.getPipelines().size());
+ Assert.assertEquals(0, datanodeMetadata.getLeaderCount());
break;
default:
Assert.fail(String.format("Datanode %s not registered",
@@ -325,7 +323,13 @@
Assert.assertEquals(2, datanodesResponse.getTotalCount());
Assert.assertEquals(2, datanodesResponse.getDatanodes().size());
- datanodesResponse.getDatanodes().forEach(this::testDatanodeResponse);
+ datanodesResponse.getDatanodes().forEach(datanodeMetadata -> {
+ try {
+ testDatanodeResponse(datanodeMetadata);
+ } catch (IOException e) {
+ Assert.fail(e.getMessage());
+ }
+ });
waitAndCheckConditionAfterHeartbeat(() -> {
Response response1 = nodeEndpoint.getDatanodes();
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestTaskStatusService.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestTaskStatusService.java
index fd792d8..9485dbf 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestTaskStatusService.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestTaskStatusService.java
@@ -20,51 +20,40 @@
import com.google.inject.AbstractModule;
import com.google.inject.Injector;
-import org.apache.hadoop.ozone.recon.persistence.AbstractSqlDatabaseTest;
-import org.hadoop.ozone.recon.schema.ReconTaskSchemaDefinition;
+
+import org.apache.hadoop.ozone.recon.persistence.AbstractReconSqlDBTest;
import org.hadoop.ozone.recon.schema.tables.daos.ReconTaskStatusDao;
import org.hadoop.ozone.recon.schema.tables.pojos.ReconTaskStatus;
-import org.jooq.Configuration;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import javax.ws.rs.core.Response;
-import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
/**
* Test for Task Status Service.
*/
-public class TestTaskStatusService extends AbstractSqlDatabaseTest {
+public class TestTaskStatusService extends AbstractReconSqlDBTest {
private TaskStatusService taskStatusService;
- private Configuration sqlConfiguration;
- private Injector childInjector;
@Before
- public void setUp() throws SQLException {
- sqlConfiguration = getInjector().getInstance((Configuration.class));
+ public void setUp() {
Injector parentInjector = getInjector();
- childInjector = parentInjector.createChildInjector(new AbstractModule() {
+ parentInjector.createChildInjector(new AbstractModule() {
@Override
protected void configure() {
taskStatusService = new TaskStatusService();
- bind(ReconTaskStatusDao.class).
- toInstance(new ReconTaskStatusDao(sqlConfiguration));
bind(TaskStatusService.class).toInstance(taskStatusService);
}
});
- ReconTaskSchemaDefinition schemaDefinition = getInjector().
- getInstance(ReconTaskSchemaDefinition.class);
- schemaDefinition.initializeSchema();
}
@Test
public void testGetTaskTimes() {
- ReconTaskStatusDao reconTaskStatusDao =
- new ReconTaskStatusDao(sqlConfiguration);
+ ReconTaskStatusDao reconTaskStatusDao = getDao(ReconTaskStatusDao.class);
ReconTaskStatus reconTaskStatusRecord = new ReconTaskStatus(
"Dummy_Task", System.currentTimeMillis(), 0L);
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestMissingContainerTask.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestMissingContainerTask.java
index 639373c..153f05a 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestMissingContainerTask.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestMissingContainerTask.java
@@ -31,37 +31,35 @@
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerManager;
import org.apache.hadoop.hdds.scm.container.ContainerReplica;
-import org.apache.hadoop.ozone.recon.persistence.AbstractSqlDatabaseTest;
+import org.apache.hadoop.ozone.recon.persistence.ContainerSchemaManager;
import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade;
+import org.apache.hadoop.ozone.recon.tasks.ReconTaskConfig;
import org.apache.hadoop.test.LambdaTestUtils;
-import org.hadoop.ozone.recon.schema.ReconTaskSchemaDefinition;
-import org.hadoop.ozone.recon.schema.UtilizationSchemaDefinition;
+import org.hadoop.ozone.recon.schema.ContainerSchemaDefinition;
+import org.hadoop.ozone.recon.schema.tables.daos.ContainerHistoryDao;
+import org.apache.hadoop.ozone.recon.persistence.AbstractReconSqlDBTest;
import org.hadoop.ozone.recon.schema.tables.daos.MissingContainersDao;
import org.hadoop.ozone.recon.schema.tables.daos.ReconTaskStatusDao;
import org.hadoop.ozone.recon.schema.tables.pojos.MissingContainers;
import org.hadoop.ozone.recon.schema.tables.pojos.ReconTaskStatus;
-import org.jooq.Configuration;
import org.junit.Assert;
import org.junit.Test;
/**
* Class to test single run of Missing Container Task.
*/
-public class TestMissingContainerTask extends AbstractSqlDatabaseTest {
+public class TestMissingContainerTask extends AbstractReconSqlDBTest {
@Test
public void testRun() throws Exception {
- Configuration sqlConfiguration =
- getInjector().getInstance((Configuration.class));
+ MissingContainersDao missingContainersTableHandle =
+ getDao(MissingContainersDao.class);
- ReconTaskSchemaDefinition taskSchemaDefinition = getInjector().getInstance(
- ReconTaskSchemaDefinition.class);
- taskSchemaDefinition.initializeSchema();
-
- UtilizationSchemaDefinition schemaDefinition =
- getInjector().getInstance(UtilizationSchemaDefinition.class);
- schemaDefinition.initializeSchema();
-
+ ContainerSchemaManager containerSchemaManager =
+ new ContainerSchemaManager(
+ mock(ContainerHistoryDao.class),
+ getSchemaDefinition(ContainerSchemaDefinition.class),
+ missingContainersTableHandle);
ReconStorageContainerManagerFacade scmMock =
mock(ReconStorageContainerManagerFacade.class);
ContainerManager containerManagerMock = mock(ContainerManager.class);
@@ -85,25 +83,21 @@
when(containerManagerMock.getContainerReplicas(new ContainerID(3L)))
.thenReturn(Collections.emptySet());
- MissingContainersDao missingContainersTableHandle =
- new MissingContainersDao(sqlConfiguration);
List<MissingContainers> all = missingContainersTableHandle.findAll();
Assert.assertTrue(all.isEmpty());
long currentTime = System.currentTimeMillis();
- ReconTaskStatusDao reconTaskStatusDao =
- new ReconTaskStatusDao(sqlConfiguration);
- MissingContainersDao missingContainersDao =
- new MissingContainersDao(sqlConfiguration);
+ ReconTaskStatusDao reconTaskStatusDao = getDao(ReconTaskStatusDao.class);
+ ReconTaskConfig reconTaskConfig = new ReconTaskConfig();
+ reconTaskConfig.setMissingContainerTaskInterval(60);
MissingContainerTask missingContainerTask =
- new MissingContainerTask(scmMock, reconTaskStatusDao,
- missingContainersDao);
- missingContainerTask.register();
+ new MissingContainerTask(scmMock.getContainerManager(),
+ reconTaskStatusDao, containerSchemaManager, reconTaskConfig);
missingContainerTask.start();
LambdaTestUtils.await(6000, 1000, () ->
- (missingContainersTableHandle.findAll().size() == 2));
- all = missingContainersTableHandle.findAll();
+ (containerSchemaManager.getAllMissingContainers().size() == 2));
+ all = containerSchemaManager.getAllMissingContainers();
// Container IDs 2 and 3 should be present in the missing containers table
Set<Long> missingContainerIDs = Collections.unmodifiableSet(
new HashSet<>(Arrays.asList(2L, 3L))
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/AbstractReconSqlDBTest.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/AbstractReconSqlDBTest.java
new file mode 100644
index 0000000..664a732
--- /dev/null
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/AbstractReconSqlDBTest.java
@@ -0,0 +1,234 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.recon.persistence;
+
+import static org.hadoop.ozone.recon.codegen.SqlDbUtils.DERBY_DRIVER_CLASS;
+
+import java.io.File;
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.sql.DataSource;
+
+import org.apache.hadoop.ozone.recon.ReconControllerModule.ReconDaoBindingModule;
+import org.apache.hadoop.ozone.recon.ReconSchemaManager;
+import org.hadoop.ozone.recon.codegen.ReconSchemaGenerationModule;
+import org.jooq.Configuration;
+import org.jooq.DSLContext;
+import org.jooq.SQLDialect;
+import org.jooq.impl.DSL;
+import org.jooq.impl.DefaultConfiguration;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.rules.TemporaryFolder;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+import com.google.inject.Provider;
+
+/**
+ * Class that provides a Recon SQL DB with all the tables created, and APIs
+ * to access the DAOs easily.
+ */
+public class AbstractReconSqlDBTest {
+
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ private Injector injector;
+ private DSLContext dslContext;
+ private Provider<DataSourceConfiguration> configurationProvider;
+
+ public AbstractReconSqlDBTest() {
+ try {
+ temporaryFolder.create();
+ configurationProvider =
+ new DerbyDataSourceConfigurationProvider(temporaryFolder.newFolder());
+ } catch (IOException e) {
+ Assert.fail();
+ }
+ }
+
+ protected AbstractReconSqlDBTest(Provider<DataSourceConfiguration> provider) {
+ try {
+ temporaryFolder.create();
+ configurationProvider = provider;
+ } catch (IOException e) {
+ Assert.fail();
+ }
+ }
+
+ @Before
+ public void createReconSchemaForTest() throws IOException {
+ injector = Guice.createInjector(getReconSqlDBModules());
+ dslContext = DSL.using(new DefaultConfiguration().set(
+ injector.getInstance(DataSource.class)));
+ createSchema(injector);
+ }
+
+ /**
+ * Get set of Guice modules needed to setup a Recon SQL DB.
+ * @return List of modules.
+ */
+ public List<Module> getReconSqlDBModules() {
+ List<Module> modules = new ArrayList<>();
+ modules.add(new JooqPersistenceModule(configurationProvider));
+ modules.add(new AbstractModule() {
+ @Override
+ protected void configure() {
+ bind(DataSourceConfiguration.class).toProvider(configurationProvider);
+ bind(ReconSchemaManager.class);
+ }
+ });
+ modules.add(new ReconSchemaGenerationModule());
+ modules.add(new ReconDaoBindingModule());
+ return modules;
+ }
+
+ /**
+ * Method to create Recon SQL schema. Used externally from ReconTestInjector.
+ * @param inj injector
+ */
+ public void createSchema(Injector inj) {
+ ReconSchemaManager reconSchemaManager =
+ inj.getInstance(ReconSchemaManager.class);
+ reconSchemaManager.createReconSchema();
+ }
+
+ protected Injector getInjector() {
+ return injector;
+ }
+
+ protected Connection getConnection() throws SQLException {
+ return injector.getInstance(DataSource.class).getConnection();
+ }
+
+ protected DSLContext getDslContext() {
+ return dslContext;
+ }
+
+ protected Configuration getConfiguration() {
+ return injector.getInstance(Configuration.class);
+ }
+
+ /**
+ * Get DAO of a specific type.
+ * @param type DAO class type.
+ * @param <T> Dao type.
+ * @return Dao instance.
+ */
+ protected <T> T getDao(Class<T> type) {
+ return injector.getInstance(type);
+ }
+
+ /**
+ * Get Schema definition of a specific type. (Essentially same as last
+ * method. Just with a different name for easy understanding.)
+ * @param type Schema definition class type.
+ * @param <T> Schema definition type.
+ * @return Schema definition instance.
+ */
+ protected <T> T getSchemaDefinition(Class<T> type) {
+ return injector.getInstance(type);
+ }
+
+ /**
+ * Local Derby datasource provider.
+ */
+ public static class DerbyDataSourceConfigurationProvider implements
+ Provider<DataSourceConfiguration> {
+
+ private final File tempDir;
+
+ public DerbyDataSourceConfigurationProvider(File tempDir) {
+ this.tempDir = tempDir;
+ }
+
+ @Override
+ public DataSourceConfiguration get() {
+ return new DataSourceConfiguration() {
+ @Override
+ public String getDriverClass() {
+ return DERBY_DRIVER_CLASS;
+ }
+
+ @Override
+ public String getJdbcUrl() {
+ return "jdbc:derby:" + tempDir.getAbsolutePath() +
+ File.separator + "derby_recon.db";
+ }
+
+ @Override
+ public String getUserName() {
+ return null;
+ }
+
+ @Override
+ public String getPassword() {
+ return null;
+ }
+
+ @Override
+ public boolean setAutoCommit() {
+ return true;
+ }
+
+ @Override
+ public long getConnectionTimeout() {
+ return 10000;
+ }
+
+ @Override
+ public String getSqlDialect() {
+ return SQLDialect.DERBY.toString();
+ }
+
+ @Override
+ public Integer getMaxActiveConnections() {
+ return 2;
+ }
+
+ @Override
+ public long getMaxConnectionAge() {
+ return 120;
+ }
+
+ @Override
+ public long getMaxIdleConnectionAge() {
+ return 120;
+ }
+
+ @Override
+ public String getConnectionTestStatement() {
+ return "SELECT 1";
+ }
+
+ @Override
+ public long getIdleConnectionTestPeriod() {
+ return 30;
+ }
+ };
+ }
+ }
+}
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/AbstractSqlDatabaseTest.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/AbstractSqlDatabaseTest.java
deleted file mode 100644
index 898dd19..0000000
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/AbstractSqlDatabaseTest.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.recon.persistence;
-
-import java.io.File;
-import java.io.IOException;
-
-import javax.sql.DataSource;
-
-import org.jooq.DSLContext;
-import org.jooq.SQLDialect;
-import org.jooq.impl.DSL;
-import org.jooq.impl.DefaultConfiguration;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.rules.TemporaryFolder;
-
-import com.google.inject.AbstractModule;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.Provider;
-
-/**
- * Create an injector for tests that need to access the SQl database.
- */
-public abstract class AbstractSqlDatabaseTest {
-
- @ClassRule
- public static TemporaryFolder temporaryFolder = new TemporaryFolder();
-
- private static Injector injector;
- private static DSLContext dslContext;
-
- @BeforeClass
- public static void setup() throws IOException {
- File tempDir = temporaryFolder.newFolder();
-
- DataSourceConfigurationProvider configurationProvider =
- new DataSourceConfigurationProvider(tempDir);
-
- JooqPersistenceModule persistenceModule =
- new JooqPersistenceModule(configurationProvider);
-
- injector = Guice.createInjector(persistenceModule, new AbstractModule() {
- @Override
- public void configure() {
- bind(DataSourceConfiguration.class).toProvider(configurationProvider);
- }
- });
- dslContext = DSL.using(new DefaultConfiguration().set(
- injector.getInstance(DataSource.class)));
- }
-
- @AfterClass
- public static void tearDown() {
- temporaryFolder.delete();
- }
-
- protected Injector getInjector() {
- return injector;
- }
-
- protected DSLContext getDslContext() {
- return dslContext;
- }
-
- /**
- * Local Sqlite datasource provider.
- */
- public static class DataSourceConfigurationProvider implements
- Provider<DataSourceConfiguration> {
-
- private final File tempDir;
-
- public DataSourceConfigurationProvider(File tempDir) {
- this.tempDir = tempDir;
- }
-
- @Override
- public DataSourceConfiguration get() {
- return new DataSourceConfiguration() {
- @Override
- public String getDriverClass() {
- return "org.sqlite.JDBC";
- }
-
- @Override
- public String getJdbcUrl() {
- return "jdbc:sqlite:" + tempDir.getAbsolutePath() +
- File.separator + "sqlite_recon.db";
- }
-
- @Override
- public String getUserName() {
- return null;
- }
-
- @Override
- public String getPassword() {
- return null;
- }
-
- @Override
- public boolean setAutoCommit() {
- return true;
- }
-
- @Override
- public long getConnectionTimeout() {
- return 10000;
- }
-
- @Override
- public String getSqlDialect() {
- return SQLDialect.SQLITE.toString();
- }
-
- @Override
- public Integer getMaxActiveConnections() {
- return 2;
- }
-
- @Override
- public Integer getMaxConnectionAge() {
- return 120;
- }
-
- @Override
- public Integer getMaxIdleConnectionAge() {
- return 120;
- }
-
- @Override
- public String getConnectionTestStatement() {
- return "SELECT 1";
- }
-
- @Override
- public Integer getIdleConnectionTestPeriod() {
- return 30;
- }
- };
- }
- }
-}
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestReconInternalSchemaDefinition.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestReconInternalSchemaDefinition.java
index 51d8f83..befd1ed 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestReconInternalSchemaDefinition.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestReconInternalSchemaDefinition.java
@@ -27,31 +27,22 @@
import java.util.ArrayList;
import java.util.List;
-import javax.sql.DataSource;
-
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
-import org.hadoop.ozone.recon.schema.ReconTaskSchemaDefinition;
import org.hadoop.ozone.recon.schema.tables.daos.ReconTaskStatusDao;
import org.hadoop.ozone.recon.schema.tables.pojos.ReconTaskStatus;
-import org.jooq.Configuration;
import org.junit.Assert;
import org.junit.Test;
/**
* Class used to test ReconInternalSchemaDefinition.
*/
-public class TestReconInternalSchemaDefinition extends AbstractSqlDatabaseTest {
+public class TestReconInternalSchemaDefinition extends AbstractReconSqlDBTest {
@Test
public void testSchemaCreated() throws Exception {
- ReconTaskSchemaDefinition schemaDefinition = getInjector().getInstance(
- ReconTaskSchemaDefinition.class);
- schemaDefinition.initializeSchema();
-
- Connection connection =
- getInjector().getInstance(DataSource.class).getConnection();
+ Connection connection = getConnection();
// Verify table definition
DatabaseMetaData metaData = connection.getMetaData();
ResultSet resultSet = metaData.getColumns(null, null,
@@ -61,9 +52,9 @@
expectedPairs.add(new ImmutablePair<>("task_name", Types.VARCHAR));
expectedPairs.add(new ImmutablePair<>("last_updated_timestamp",
- Types.INTEGER));
+ Types.BIGINT));
expectedPairs.add(new ImmutablePair<>("last_updated_seq_number",
- Types.INTEGER));
+ Types.BIGINT));
List<Pair<String, Integer>> actualPairs = new ArrayList<>();
@@ -80,14 +71,7 @@
@Test
public void testReconTaskStatusCRUDOperations() throws Exception {
// Verify table exists
- ReconTaskSchemaDefinition schemaDefinition = getInjector().getInstance(
- ReconTaskSchemaDefinition.class);
-
- schemaDefinition.initializeSchema();
-
- DataSource ds = getInjector().getInstance(DataSource.class);
- Connection connection = ds.getConnection();
-
+ Connection connection = getConnection();
DatabaseMetaData metaData = connection.getMetaData();
ResultSet resultSet = metaData.getTables(null, null,
RECON_TASK_STATUS_TABLE_NAME, null);
@@ -97,9 +81,7 @@
resultSet.getString("TABLE_NAME"));
}
- ReconTaskStatusDao dao = new ReconTaskStatusDao(getInjector().getInstance(
- Configuration.class));
-
+ ReconTaskStatusDao dao = getDao(ReconTaskStatusDao.class);
long now = System.currentTimeMillis();
ReconTaskStatus newRecord = new ReconTaskStatus();
newRecord.setTaskName("HelloWorldTask");
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestReconWithDifferentSqlDBs.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestReconWithDifferentSqlDBs.java
new file mode 100644
index 0000000..12b9659
--- /dev/null
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestReconWithDifferentSqlDBs.java
@@ -0,0 +1,159 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.recon.persistence;
+
+import static java.util.stream.Collectors.toList;
+import static org.apache.hadoop.ozone.recon.ReconControllerModule.ReconDaoBindingModule.RECON_DAO_LIST;
+import static org.hadoop.ozone.recon.codegen.SqlDbUtils.SQLITE_DRIVER_CLASS;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import java.io.File;
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.stream.Stream;
+
+import org.hadoop.ozone.recon.schema.tables.daos.ReconTaskStatusDao;
+import org.hadoop.ozone.recon.schema.tables.pojos.ReconTaskStatus;
+import org.jooq.SQLDialect;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import com.google.inject.Provider;
+
+/**
+ * Test Recon schema with different DBs.
+ */
+@RunWith(Parameterized.class)
+public class TestReconWithDifferentSqlDBs extends AbstractReconSqlDBTest {
+
+ public TestReconWithDifferentSqlDBs(
+ Provider<DataSourceConfiguration> provider) {
+ super(provider);
+ }
+
+ @Parameterized.Parameters(name = "{0}")
+ public static Iterable<Object[]> parameters() throws IOException {
+ TemporaryFolder temporaryFolder = new TemporaryFolder();
+ temporaryFolder.create();
+ return Stream.of(
+ new DerbyDataSourceConfigurationProvider(temporaryFolder.newFolder()),
+ new SqliteDataSourceConfigurationProvider(temporaryFolder.newFolder()))
+ .map(each -> new Object[] {each})
+ .collect(toList());
+ }
+
+ /**
+ * Make sure schema was created correctly.
+ * @throws SQLException
+ */
+ @Test
+ public void testSchemaSetup() throws SQLException {
+ assertNotNull(getInjector());
+ assertNotNull(getConfiguration());
+ assertNotNull(getDslContext());
+ assertNotNull(getConnection());
+ RECON_DAO_LIST.forEach(dao -> {
+ assertNotNull(getDao(dao));
+ });
+ ReconTaskStatusDao dao = getDao(ReconTaskStatusDao.class);
+ dao.insert(new ReconTaskStatus("TestTask", 1L, 2L));
+ assertEquals(1, dao.findAll().size());
+ }
+
+ /**
+ * Local Sqlite datasource provider.
+ */
+ public static class SqliteDataSourceConfigurationProvider implements
+ Provider<DataSourceConfiguration> {
+
+ private final File tempDir;
+
+ public SqliteDataSourceConfigurationProvider(File tempDir) {
+ this.tempDir = tempDir;
+ }
+
+ @Override
+ public DataSourceConfiguration get() {
+ return new DataSourceConfiguration() {
+ @Override
+ public String getDriverClass() {
+ return SQLITE_DRIVER_CLASS;
+ }
+
+ @Override
+ public String getJdbcUrl() {
+ return "jdbc:sqlite:" + tempDir.getAbsolutePath() +
+ File.separator + "recon_sqlite.db";
+ }
+
+ @Override
+ public String getUserName() {
+ return null;
+ }
+
+ @Override
+ public String getPassword() {
+ return null;
+ }
+
+ @Override
+ public boolean setAutoCommit() {
+ return true;
+ }
+
+ @Override
+ public long getConnectionTimeout() {
+ return 10000;
+ }
+
+ @Override
+ public String getSqlDialect() {
+ return SQLDialect.SQLITE.toString();
+ }
+
+ @Override
+ public Integer getMaxActiveConnections() {
+ return 2;
+ }
+
+ @Override
+ public long getMaxConnectionAge() {
+ return 120;
+ }
+
+ @Override
+ public long getMaxIdleConnectionAge() {
+ return 120;
+ }
+
+ @Override
+ public String getConnectionTestStatement() {
+ return "SELECT 1";
+ }
+
+ @Override
+ public long getIdleConnectionTestPeriod() {
+ return 30;
+ }
+ };
+ }
+ }
+}
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestSqlSchemaSetup.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestSqlSchemaSetup.java
new file mode 100644
index 0000000..19f8d70
--- /dev/null
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestSqlSchemaSetup.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.persistence;
+
+import static org.apache.hadoop.ozone.recon.ReconControllerModule.ReconDaoBindingModule.RECON_DAO_LIST;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import java.sql.SQLException;
+
+import org.hadoop.ozone.recon.schema.tables.daos.ReconTaskStatusDao;
+import org.hadoop.ozone.recon.schema.tables.pojos.ReconTaskStatus;
+import org.junit.Test;
+
+/**
+ * Class to test basic SQL schema setup.
+ */
+public class TestSqlSchemaSetup extends AbstractReconSqlDBTest {
+
+ /**
+ * Make sure schema was created correctly.
+ * @throws SQLException
+ */
+ @Test
+ public void testSchemaSetup() throws SQLException {
+ assertNotNull(getInjector());
+ assertNotNull(getConfiguration());
+ assertNotNull(getDslContext());
+ assertNotNull(getConnection());
+ RECON_DAO_LIST.forEach(dao -> {
+ assertNotNull(getDao(dao));
+ });
+ ReconTaskStatusDao dao = getDao(ReconTaskStatusDao.class);
+ dao.insert(new ReconTaskStatus("TestTask", 1L, 2L));
+ assertEquals(1, dao.findAll().size());
+ }
+}
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestStatsSchemaDefinition.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestStatsSchemaDefinition.java
index 864e59e..af08383 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestStatsSchemaDefinition.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestStatsSchemaDefinition.java
@@ -19,14 +19,11 @@
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
-import org.hadoop.ozone.recon.schema.StatsSchemaDefinition;
import org.hadoop.ozone.recon.schema.tables.daos.GlobalStatsDao;
import org.hadoop.ozone.recon.schema.tables.pojos.GlobalStats;
-import org.jooq.Configuration;
import org.junit.Assert;
import org.junit.Test;
-import javax.sql.DataSource;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
@@ -40,17 +37,11 @@
/**
* Class used to test StatsSchemaDefinition.
*/
-public class TestStatsSchemaDefinition extends AbstractSqlDatabaseTest {
+public class TestStatsSchemaDefinition extends AbstractReconSqlDBTest {
@Test
public void testIfStatsSchemaCreated() throws Exception {
- StatsSchemaDefinition schemaDefinition = getInjector().getInstance(
- StatsSchemaDefinition.class);
-
- schemaDefinition.initializeSchema();
-
- Connection connection =
- getInjector().getInstance(DataSource.class).getConnection();
+ Connection connection = getConnection();
// Verify table definition
DatabaseMetaData metaData = connection.getMetaData();
ResultSet resultSet = metaData.getColumns(null, null,
@@ -59,9 +50,9 @@
List<Pair<String, Integer>> expectedPairs = new ArrayList<>();
expectedPairs.add(new ImmutablePair<>("key", Types.VARCHAR));
- expectedPairs.add(new ImmutablePair<>("value", Types.INTEGER));
+ expectedPairs.add(new ImmutablePair<>("value", Types.BIGINT));
expectedPairs.add(new ImmutablePair<>("last_updated_timestamp",
- Types.VARCHAR));
+ Types.TIMESTAMP));
List<Pair<String, Integer>> actualPairs = new ArrayList<>();
@@ -76,14 +67,7 @@
@Test
public void testGlobalStatsCRUDOperations() throws Exception {
- // Verify table exists
- StatsSchemaDefinition schemaDefinition = getInjector().getInstance(
- StatsSchemaDefinition.class);
-
- schemaDefinition.initializeSchema();
-
- DataSource ds = getInjector().getInstance(DataSource.class);
- Connection connection = ds.getConnection();
+ Connection connection = getConnection();
DatabaseMetaData metaData = connection.getMetaData();
ResultSet resultSet = metaData.getTables(null, null,
@@ -94,8 +78,7 @@
resultSet.getString("TABLE_NAME"));
}
- GlobalStatsDao dao = new GlobalStatsDao(
- getInjector().getInstance(Configuration.class));
+ GlobalStatsDao dao = getDao(GlobalStatsDao.class);
long now = System.currentTimeMillis();
GlobalStats newRecord = new GlobalStats();
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestUtilizationSchemaDefinition.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestUtilizationSchemaDefinition.java
index 22cc55b..9e781da 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestUtilizationSchemaDefinition.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestUtilizationSchemaDefinition.java
@@ -31,17 +31,13 @@
import java.util.ArrayList;
import java.util.List;
-import javax.sql.DataSource;
-
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
-import org.hadoop.ozone.recon.schema.UtilizationSchemaDefinition;
import org.hadoop.ozone.recon.schema.tables.daos.ClusterGrowthDailyDao;
import org.hadoop.ozone.recon.schema.tables.daos.FileCountBySizeDao;
import org.hadoop.ozone.recon.schema.tables.pojos.ClusterGrowthDaily;
import org.hadoop.ozone.recon.schema.tables.pojos.FileCountBySize;
import org.hadoop.ozone.recon.schema.tables.records.FileCountBySizeRecord;
-import org.jooq.Configuration;
import org.jooq.Table;
import org.jooq.UniqueKey;
import org.junit.Assert;
@@ -50,17 +46,11 @@
/**
* Test persistence module provides connection and transaction awareness.
*/
-public class TestUtilizationSchemaDefinition extends AbstractSqlDatabaseTest {
+public class TestUtilizationSchemaDefinition extends AbstractReconSqlDBTest {
@Test
public void testReconSchemaCreated() throws Exception {
- UtilizationSchemaDefinition schemaDefinition = getInjector().getInstance(
- UtilizationSchemaDefinition.class);
-
- schemaDefinition.initializeSchema();
-
- Connection connection =
- getInjector().getInstance(DataSource.class).getConnection();
+ Connection connection = getConnection();
// Verify table definition
DatabaseMetaData metaData = connection.getMetaData();
ResultSet resultSet = metaData.getColumns(null, null,
@@ -68,12 +58,12 @@
List<Pair<String, Integer>> expectedPairs = new ArrayList<>();
- expectedPairs.add(new ImmutablePair<>("timestamp", Types.VARCHAR));
+ expectedPairs.add(new ImmutablePair<>("timestamp", Types.TIMESTAMP));
expectedPairs.add(new ImmutablePair<>("datanode_id", Types.INTEGER));
expectedPairs.add(new ImmutablePair<>("datanode_host", Types.VARCHAR));
expectedPairs.add(new ImmutablePair<>("rack_id", Types.VARCHAR));
- expectedPairs.add(new ImmutablePair<>("available_size", Types.INTEGER));
- expectedPairs.add(new ImmutablePair<>("used_size", Types.INTEGER));
+ expectedPairs.add(new ImmutablePair<>("available_size", Types.BIGINT));
+ expectedPairs.add(new ImmutablePair<>("used_size", Types.BIGINT));
expectedPairs.add(new ImmutablePair<>("container_count", Types.INTEGER));
expectedPairs.add(new ImmutablePair<>("block_count", Types.INTEGER));
@@ -92,9 +82,9 @@
List<Pair<String, Integer>> expectedPairsFileCount = new ArrayList<>();
expectedPairsFileCount.add(
- new ImmutablePair<>("file_size", Types.INTEGER));
+ new ImmutablePair<>("file_size", Types.BIGINT));
expectedPairsFileCount.add(
- new ImmutablePair<>("count", Types.INTEGER));
+ new ImmutablePair<>("count", Types.BIGINT));
List<Pair<String, Integer>> actualPairsFileCount = new ArrayList<>();
while(resultSetFileCount.next()) {
@@ -111,12 +101,7 @@
@Test
public void testClusterGrowthDailyCRUDOperations() throws Exception {
// Verify table exists
- UtilizationSchemaDefinition schemaDefinition = getInjector().getInstance(
- UtilizationSchemaDefinition.class);
- schemaDefinition.initializeSchema();
-
- DataSource ds = getInjector().getInstance(DataSource.class);
- Connection connection = ds.getConnection();
+ Connection connection = getConnection();
DatabaseMetaData metaData = connection.getMetaData();
ResultSet resultSet = metaData.getTables(null, null,
@@ -127,9 +112,7 @@
resultSet.getString("TABLE_NAME"));
}
- ClusterGrowthDailyDao dao = new ClusterGrowthDailyDao(
- getInjector().getInstance(Configuration.class));
-
+ ClusterGrowthDailyDao dao = getDao(ClusterGrowthDailyDao.class);
long now = System.currentTimeMillis();
ClusterGrowthDaily newRecord = new ClusterGrowthDaily();
newRecord.setTimestamp(new Timestamp(now));
@@ -187,12 +170,7 @@
@Test
public void testFileCountBySizeCRUDOperations() throws SQLException {
- UtilizationSchemaDefinition schemaDefinition = getInjector().getInstance(
- UtilizationSchemaDefinition.class);
- schemaDefinition.initializeSchema();
-
- DataSource ds = getInjector().getInstance(DataSource.class);
- Connection connection = ds.getConnection();
+ Connection connection = getConnection();
DatabaseMetaData metaData = connection.getMetaData();
ResultSet resultSet = metaData.getTables(null, null,
@@ -203,8 +181,7 @@
resultSet.getString("TABLE_NAME"));
}
- FileCountBySizeDao fileCountBySizeDao = new FileCountBySizeDao(
- getInjector().getInstance(Configuration.class));
+ FileCountBySizeDao fileCountBySizeDao = getDao(FileCountBySizeDao.class);
FileCountBySize newRecord = new FileCountBySize();
newRecord.setFileSize(1024L);
@@ -221,8 +198,6 @@
dbRecord = fileCountBySizeDao.findById(1024L);
assertEquals(Long.valueOf(2), dbRecord.getCount());
-
-
Table<FileCountBySizeRecord> fileCountBySizeRecordTable =
fileCountBySizeDao.getTable();
List<UniqueKey<FileCountBySizeRecord>> tableKeys =
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/AbstractReconContainerManagerTest.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/AbstractReconContainerManagerTest.java
index fce22b2..04010e5 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/AbstractReconContainerManagerTest.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/AbstractReconContainerManagerTest.java
@@ -18,15 +18,6 @@
package org.apache.hadoop.ozone.recon.scm;
-import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.OPEN;
-import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
-import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.STAND_ALONE;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
-import static org.apache.hadoop.ozone.recon.AbstractOMMetadataManagerTest.getRandomPipeline;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
import java.io.IOException;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -40,11 +31,23 @@
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.ozone.recon.persistence.ContainerSchemaManager;
import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider;
+
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.OPEN;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.STAND_ALONE;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandomPipeline;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.rules.TemporaryFolder;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
/**
* Abstract class for Recon Container Manager related tests.
@@ -58,6 +61,7 @@
private SCMStorageConfig scmStorageConfig;
private ReconPipelineManager pipelineManager;
private ReconContainerManager containerManager;
+ private DBStore store;
@Before
public void setUp() throws Exception {
@@ -65,20 +69,28 @@
conf.set(OZONE_METADATA_DIRS,
temporaryFolder.newFolder().getAbsolutePath());
conf.set(OZONE_SCM_NAMES, "localhost");
+ store = DBStoreBuilder.createDBStore(conf, new ReconDBDefinition());
scmStorageConfig = new ReconStorageConfig(conf);
NetworkTopology clusterMap = new NetworkTopologyImpl(conf);
EventQueue eventQueue = new EventQueue();
NodeManager nodeManager =
new SCMNodeManager(conf, scmStorageConfig, eventQueue, clusterMap);
- pipelineManager = new ReconPipelineManager(conf, nodeManager, eventQueue);
- containerManager = new ReconContainerManager(conf, pipelineManager,
- getScmServiceProvider());
+ pipelineManager = new ReconPipelineManager(conf, nodeManager,
+ ReconDBDefinition.PIPELINES.getTable(store), eventQueue);
+ containerManager = new ReconContainerManager(
+ conf,
+ ReconDBDefinition.CONTAINERS.getTable(store),
+ store,
+ pipelineManager,
+ getScmServiceProvider(),
+ mock(ContainerSchemaManager.class));
}
@After
- public void tearDown() throws IOException {
+ public void tearDown() throws Exception {
containerManager.close();
pipelineManager.close();
+ store.close();
}
protected OzoneConfiguration getConf() {
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java
index 5fd6389..ccc1c80 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java
@@ -22,7 +22,7 @@
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.OPEN;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.STAND_ALONE;
-import static org.apache.hadoop.ozone.recon.AbstractOMMetadataManagerTest.getRandomPipeline;
+import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandomPipeline;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconPipelineManager.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconPipelineManager.java
index be5ee07..c891f33 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconPipelineManager.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconPipelineManager.java
@@ -18,15 +18,6 @@
package org.apache.hadoop.ozone.recon.scm;
-import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
-import static org.apache.hadoop.ozone.recon.AbstractOMMetadataManagerTest.getRandomPipeline;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
-
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
@@ -45,11 +36,23 @@
import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
import org.apache.hadoop.ozone.recon.scm.ReconPipelineFactory.ReconPipelineProvider;
+
+import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandomPipeline;
+import org.junit.After;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
+import static org.mockito.Mockito.mock;
/**
* Class to test Recon Pipeline Manager.
@@ -61,6 +64,7 @@
private OzoneConfiguration conf;
private SCMStorageConfig scmStorageConfig;
+ private DBStore store;
@Before
public void setup() throws IOException {
@@ -69,6 +73,12 @@
temporaryFolder.newFolder().getAbsolutePath());
conf.set(OZONE_SCM_NAMES, "localhost");
scmStorageConfig = new ReconStorageConfig(conf);
+ store = DBStoreBuilder.createDBStore(conf, new ReconDBDefinition());
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ store.close();
}
@Test
@@ -103,7 +113,8 @@
new SCMNodeManager(conf, scmStorageConfig, eventQueue, clusterMap);
try (ReconPipelineManager reconPipelineManager =
- new ReconPipelineManager(conf, nodeManager, eventQueue)) {
+ new ReconPipelineManager(conf, nodeManager,
+ ReconDBDefinition.PIPELINES.getTable(store), eventQueue)) {
reconPipelineManager.addPipeline(validPipeline);
reconPipelineManager.addPipeline(invalidPipeline);
@@ -138,7 +149,8 @@
new SCMNodeManager(conf, scmStorageConfig, eventQueue, clusterMap);
ReconPipelineManager reconPipelineManager =
- new ReconPipelineManager(conf, nodeManager, eventQueue);
+ new ReconPipelineManager(conf, nodeManager,
+ ReconDBDefinition.PIPELINES.getTable(store), eventQueue);
assertFalse(reconPipelineManager.containsPipeline(pipeline.getId()));
reconPipelineManager.addPipeline(pipeline);
assertTrue(reconPipelineManager.containsPipeline(pipeline.getId()));
@@ -150,7 +162,8 @@
NodeManager nodeManagerMock = mock(NodeManager.class);
ReconPipelineManager reconPipelineManager = new ReconPipelineManager(
- conf, nodeManagerMock, new EventQueue());
+ conf, nodeManagerMock, ReconDBDefinition.PIPELINES.getTable(store),
+ new EventQueue());
PipelineFactory pipelineFactory = reconPipelineManager.getPipelineFactory();
assertTrue(pipelineFactory instanceof ReconPipelineFactory);
ReconPipelineFactory reconPipelineFactory =
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconPipelineReportHandler.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconPipelineReportHandler.java
index e04101b..e72df36 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconPipelineReportHandler.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconPipelineReportHandler.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.ozone.recon.scm;
-import static org.apache.hadoop.ozone.recon.AbstractOMMetadataManagerTest.getRandomPipeline;
+import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandomPipeline;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestContainerDBServiceProviderImpl.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestContainerDBServiceProviderImpl.java
index 2392f8a..727cfe3 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestContainerDBServiceProviderImpl.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestContainerDBServiceProviderImpl.java
@@ -26,13 +26,10 @@
import java.util.HashMap;
import java.util.Map;
-import org.apache.hadoop.ozone.recon.GuiceInjectorUtilsForTestsImpl;
+import org.apache.hadoop.ozone.recon.ReconTestInjector;
import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider;
-import org.hadoop.ozone.recon.schema.StatsSchemaDefinition;
-import org.jooq.impl.DSL;
-import org.jooq.impl.DefaultConfiguration;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -40,10 +37,6 @@
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
-import com.google.inject.Injector;
-
-import javax.sql.DataSource;
-
/**
* Unit Tests for ContainerDBServiceProviderImpl.
*/
@@ -52,14 +45,28 @@
@ClassRule
public static TemporaryFolder tempFolder = new TemporaryFolder();
private static ContainerDBServiceProvider containerDbServiceProvider;
- private static Injector injector;
- private static GuiceInjectorUtilsForTestsImpl guiceInjectorTest =
- new GuiceInjectorUtilsForTestsImpl();
private String keyPrefix1 = "V3/B1/K1";
private String keyPrefix2 = "V3/B1/K2";
private String keyPrefix3 = "V3/B2/K1";
+ @BeforeClass
+ public static void setupOnce() throws Exception {
+ ReconTestInjector reconTestInjector =
+ new ReconTestInjector.Builder(tempFolder)
+ .withReconSqlDb()
+ .withContainerDB()
+ .build();
+ containerDbServiceProvider =
+ reconTestInjector.getInstance(ContainerDBServiceProvider.class);
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ // Reset containerDB before running each test
+ containerDbServiceProvider.initNewContainerDB(null);
+ }
+
private void populateKeysInContainers(long containerId1, long containerId2)
throws Exception {
@@ -80,33 +87,6 @@
3);
}
- private static void initializeInjector() throws Exception {
- injector = guiceInjectorTest.getInjector(
- null, null, tempFolder);
- }
-
- @BeforeClass
- public static void setupOnce() throws Exception {
-
- initializeInjector();
-
- DSL.using(new DefaultConfiguration().set(
- injector.getInstance(DataSource.class)));
-
- containerDbServiceProvider = injector.getInstance(
- ContainerDBServiceProvider.class);
-
- StatsSchemaDefinition schemaDefinition = injector.getInstance(
- StatsSchemaDefinition.class);
- schemaDefinition.initializeSchema();
- }
-
- @Before
- public void setUp() throws Exception {
- // Reset containerDB before running each test
- containerDbServiceProvider.initNewContainerDB(null);
- }
-
@Test
public void testInitNewContainerDB() throws Exception {
long containerId = System.currentTimeMillis();
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestOzoneManagerServiceProviderImpl.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestOzoneManagerServiceProviderImpl.java
index 3014f35..e989914 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestOzoneManagerServiceProviderImpl.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestOzoneManagerServiceProviderImpl.java
@@ -18,6 +18,10 @@
package org.apache.hadoop.ozone.recon.spi.impl;
+import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager;
+import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.initializeEmptyOmMetadataManager;
+import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.initializeNewOmMetadataManager;
+import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDataToOm;
import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_DB_DIR;
import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_OM_SNAPSHOT_DB_DIR;
import static org.apache.hadoop.ozone.recon.ReconUtils.createTarFile;
@@ -27,6 +31,7 @@
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doCallRealMethod;
import static org.mockito.Mockito.doNothing;
@@ -44,23 +49,26 @@
import java.nio.file.Paths;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.helpers.DBUpdates;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
-import org.apache.hadoop.ozone.recon.AbstractOMMetadataManagerTest;
import org.apache.hadoop.ozone.recon.ReconUtils;
+import org.apache.hadoop.ozone.recon.metrics.OzoneManagerSyncMetrics;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
import org.apache.hadoop.ozone.recon.tasks.OMDBUpdatesHandler;
import org.apache.hadoop.ozone.recon.tasks.OMUpdateEventBatch;
import org.apache.hadoop.ozone.recon.tasks.ReconTaskController;
-import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
-import org.apache.hadoop.hdds.utils.db.DBUpdatesWrapper;
-import org.apache.hadoop.hdds.utils.db.RDBStore;
+
import org.hadoop.ozone.recon.schema.tables.daos.ReconTaskStatusDao;
import org.hadoop.ozone.recon.schema.tables.pojos.ReconTaskStatus;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import org.mockito.ArgumentCaptor;
import org.rocksdb.RocksDB;
import org.rocksdb.TransactionLogIterator;
@@ -69,8 +77,10 @@
/**
* Class to test Ozone Manager Service Provider Implementation.
*/
-public class TestOzoneManagerServiceProviderImpl extends
- AbstractOMMetadataManagerTest {
+public class TestOzoneManagerServiceProviderImpl {
+
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
private OzoneConfiguration configuration;
private OzoneManagerProtocol ozoneManagerProtocol;
@@ -83,15 +93,17 @@
configuration.set(OZONE_RECON_DB_DIR,
temporaryFolder.newFolder().getAbsolutePath());
configuration.set("ozone.om.address", "localhost:9862");
- ozoneManagerProtocol = getMockOzoneManagerClient(new DBUpdatesWrapper());
+ ozoneManagerProtocol = getMockOzoneManagerClient(new DBUpdates());
}
@Test
public void testUpdateReconOmDBWithNewSnapshot() throws Exception {
- OMMetadataManager omMetadataManager = initializeNewOmMetadataManager();
+ OMMetadataManager omMetadataManager =
+ initializeNewOmMetadataManager(temporaryFolder.newFolder());
ReconOMMetadataManager reconOMMetadataManager =
- getTestMetadataManager(omMetadataManager);
+ getTestReconOmMetadataManager(omMetadataManager,
+ temporaryFolder.newFolder());
writeDataToOm(omMetadataManager, "key_one");
writeDataToOm(omMetadataManager, "key_two");
@@ -101,7 +113,7 @@
File tarFile = createTarFile(checkpoint.getCheckpointLocation());
InputStream inputStream = new FileInputStream(tarFile);
ReconUtils reconUtilsMock = getMockReconUtils();
- when(reconUtilsMock.makeHttpCall(any(), anyString()))
+ when(reconUtilsMock.makeHttpCall(any(), anyString(), anyBoolean()))
.thenReturn(inputStream);
ReconTaskController reconTaskController = getMockTaskController();
@@ -152,7 +164,7 @@
File tarFile = createTarFile(checkpointDir.toPath());
InputStream fileInputStream = new FileInputStream(tarFile);
ReconUtils reconUtilsMock = getMockReconUtils();
- when(reconUtilsMock.makeHttpCall(any(), anyString()))
+ when(reconUtilsMock.makeHttpCall(any(), anyString(), anyBoolean()))
.thenReturn(fileInputStream);
ReconOMMetadataManager reconOMMetadataManager =
@@ -176,13 +188,14 @@
// Writing 2 Keys into a source OM DB and collecting it in a
// DBUpdatesWrapper.
- OMMetadataManager sourceOMMetadataMgr = initializeNewOmMetadataManager();
+ OMMetadataManager sourceOMMetadataMgr =
+ initializeNewOmMetadataManager(temporaryFolder.newFolder());
writeDataToOm(sourceOMMetadataMgr, "key_one");
writeDataToOm(sourceOMMetadataMgr, "key_two");
RocksDB rocksDB = ((RDBStore)sourceOMMetadataMgr.getStore()).getDb();
TransactionLogIterator transactionLogIterator = rocksDB.getUpdatesSince(0L);
- DBUpdatesWrapper dbUpdatesWrapper = new DBUpdatesWrapper();
+ DBUpdates dbUpdatesWrapper = new DBUpdates();
while(transactionLogIterator.isValid()) {
TransactionLogIterator.BatchResult result =
transactionLogIterator.getBatch();
@@ -194,11 +207,13 @@
}
// OM Service Provider's Metadata Manager.
- OMMetadataManager omMetadataManager = initializeNewOmMetadataManager();
+ OMMetadataManager omMetadataManager =
+ initializeNewOmMetadataManager(temporaryFolder.newFolder());
OzoneManagerServiceProviderImpl ozoneManagerServiceProvider =
new OzoneManagerServiceProviderImpl(configuration,
- getTestMetadataManager(omMetadataManager),
+ getTestReconOmMetadataManager(omMetadataManager,
+ temporaryFolder.newFolder()),
getMockTaskController(), new ReconUtils(),
getMockOzoneManagerClient(dbUpdatesWrapper));
@@ -207,6 +222,11 @@
ozoneManagerServiceProvider.getAndApplyDeltaUpdatesFromOM(
0L, updatesHandler);
+ OzoneManagerSyncMetrics metrics = ozoneManagerServiceProvider.getMetrics();
+ assertEquals(4.0,
+ metrics.getAverageNumUpdatesInDeltaRequest().value(), 0.0);
+ assertEquals(1, metrics.getNumNonZeroDeltaRequests().value());
+
// In this method, we have to assert the "GET" part and the "APPLY" path.
// Assert GET path --> verify if the OMDBUpdatesHandler picked up the 4
@@ -229,8 +249,9 @@
public void testSyncDataFromOMFullSnapshot() throws Exception {
// Empty OM DB to start with.
- ReconOMMetadataManager omMetadataManager = getTestMetadataManager(
- initializeEmptyOmMetadataManager());
+ ReconOMMetadataManager omMetadataManager = getTestReconOmMetadataManager(
+ initializeEmptyOmMetadataManager(temporaryFolder.newFolder()),
+ temporaryFolder.newFolder());
ReconTaskStatusDao reconTaskStatusDaoMock =
mock(ReconTaskStatusDao.class);
doNothing().when(reconTaskStatusDaoMock)
@@ -246,6 +267,9 @@
new MockOzoneServiceProvider(configuration, omMetadataManager,
reconTaskControllerMock, new ReconUtils(), ozoneManagerProtocol);
+ OzoneManagerSyncMetrics metrics = ozoneManagerServiceProvider.getMetrics();
+ assertEquals(0, metrics.getNumSnapshotRequests().value());
+
// Should trigger full snapshot request.
ozoneManagerServiceProvider.syncDataFromOM();
@@ -257,14 +281,16 @@
.equals(OmSnapshotRequest.name()));
verify(reconTaskControllerMock, times(1))
.reInitializeTasks(omMetadataManager);
+ assertEquals(1, metrics.getNumSnapshotRequests().value());
}
@Test
public void testSyncDataFromOMDeltaUpdates() throws Exception {
// Non-Empty OM DB to start with.
- ReconOMMetadataManager omMetadataManager = getTestMetadataManager(
- initializeNewOmMetadataManager());
+ ReconOMMetadataManager omMetadataManager = getTestReconOmMetadataManager(
+ initializeNewOmMetadataManager(temporaryFolder.newFolder()),
+ temporaryFolder.newFolder());
ReconTaskStatusDao reconTaskStatusDaoMock =
mock(ReconTaskStatusDao.class);
doNothing().when(reconTaskStatusDaoMock)
@@ -281,6 +307,8 @@
new OzoneManagerServiceProviderImpl(configuration, omMetadataManager,
reconTaskControllerMock, new ReconUtils(), ozoneManagerProtocol);
+ OzoneManagerSyncMetrics metrics = ozoneManagerServiceProvider.getMetrics();
+
// Should trigger delta updates.
ozoneManagerServiceProvider.syncDataFromOM();
@@ -293,6 +321,7 @@
verify(reconTaskControllerMock, times(1))
.consumeOMEvents(any(OMUpdateEventBatch.class),
any(OMMetadataManager.class));
+ assertEquals(0, metrics.getNumSnapshotRequests().value());
}
private ReconTaskController getMockTaskController() {
@@ -309,7 +338,7 @@
}
private OzoneManagerProtocol getMockOzoneManagerClient(
- DBUpdatesWrapper dbUpdatesWrapper) throws IOException {
+ DBUpdates dbUpdatesWrapper) throws IOException {
OzoneManagerProtocol ozoneManagerProtocolMock =
mock(OzoneManagerProtocol.class);
when(ozoneManagerProtocolMock.getDBUpdates(any(OzoneManagerProtocolProtos
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestContainerKeyMapperTask.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestContainerKeyMapperTask.java
index 8634998..cb6f5b3 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestContainerKeyMapperTask.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestContainerKeyMapperTask.java
@@ -18,6 +18,11 @@
package org.apache.hadoop.ozone.recon.tasks;
+import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getOmKeyLocationInfo;
+import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandomPipeline;
+import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager;
+import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.initializeNewOmMetadataManager;
+import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDataToOm;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
@@ -36,69 +41,47 @@
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
-import org.apache.hadoop.ozone.recon.AbstractOMMetadataManagerTest;
-import org.apache.hadoop.ozone.recon.GuiceInjectorUtilsForTestsImpl;
+import org.apache.hadoop.ozone.recon.ReconTestInjector;
import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider;
import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl;
import org.apache.hadoop.hdds.utils.db.Table;
-import org.hadoop.ozone.recon.schema.StatsSchemaDefinition;
-import org.jooq.impl.DSL;
-import org.jooq.impl.DefaultConfiguration;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
-import com.google.inject.Injector;
-import javax.sql.DataSource;
+import org.junit.rules.TemporaryFolder;
/**
* Unit test for Container Key mapper task.
*/
-public class TestContainerKeyMapperTask extends AbstractOMMetadataManagerTest {
+public class TestContainerKeyMapperTask {
+
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
private ContainerDBServiceProvider containerDbServiceProvider;
private OMMetadataManager omMetadataManager;
private ReconOMMetadataManager reconOMMetadataManager;
- private Injector injector;
private OzoneManagerServiceProviderImpl ozoneManagerServiceProvider;
- private boolean setUpIsDone = false;
- private GuiceInjectorUtilsForTestsImpl guiceInjectorTest =
- new GuiceInjectorUtilsForTestsImpl();
-
- private Injector getInjector() {
- return injector;
- }
-
- private void initializeInjector() throws Exception {
- omMetadataManager = initializeNewOmMetadataManager();
- ozoneManagerServiceProvider = getMockOzoneManagerServiceProvider();
- reconOMMetadataManager = getTestMetadataManager(omMetadataManager);
-
- injector = guiceInjectorTest.getInjector(
- ozoneManagerServiceProvider, reconOMMetadataManager, temporaryFolder);
- }
@Before
public void setUp() throws Exception {
- // The following setup is run only once
- if (!setUpIsDone) {
- initializeInjector();
+ omMetadataManager = initializeNewOmMetadataManager(
+ temporaryFolder.newFolder());
+ ozoneManagerServiceProvider = getMockOzoneManagerServiceProvider();
+ reconOMMetadataManager = getTestReconOmMetadataManager(omMetadataManager,
+ temporaryFolder.newFolder());
- DSL.using(new DefaultConfiguration().set(
- injector.getInstance(DataSource.class)));
-
- containerDbServiceProvider = injector.getInstance(
- ContainerDBServiceProvider.class);
-
- StatsSchemaDefinition schemaDefinition = getInjector().getInstance(
- StatsSchemaDefinition.class);
- schemaDefinition.initializeSchema();
-
- setUpIsDone = true;
- }
-
- containerDbServiceProvider = injector.getInstance(
- ContainerDBServiceProvider.class);
+ ReconTestInjector reconTestInjector =
+ new ReconTestInjector.Builder(temporaryFolder)
+ .withReconSqlDb()
+ .withReconOm(reconOMMetadataManager)
+ .withOmServiceProvider(ozoneManagerServiceProvider)
+ .withContainerDB()
+ .build();
+ containerDbServiceProvider =
+ reconTestInjector.getInstance(ContainerDBServiceProvider.class);
}
@Test
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java
index b63ddd7..e0e37b5 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java
@@ -23,16 +23,13 @@
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.hdds.utils.db.TypedTable;
-import org.apache.hadoop.ozone.recon.persistence.AbstractSqlDatabaseTest;
+import org.apache.hadoop.ozone.recon.persistence.AbstractReconSqlDBTest;
import org.apache.hadoop.ozone.recon.tasks.OMDBUpdateEvent.OMUpdateEventBuilder;
-import org.hadoop.ozone.recon.schema.UtilizationSchemaDefinition;
import org.hadoop.ozone.recon.schema.tables.daos.FileCountBySizeDao;
-import org.jooq.Configuration;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
-import java.sql.SQLException;
import java.util.Arrays;
import static org.apache.hadoop.ozone.recon.tasks.OMDBUpdateEvent.OMDBUpdateAction.DELETE;
@@ -49,18 +46,13 @@
/**
* Unit test for File Size Count Task.
*/
-public class TestFileSizeCountTask extends AbstractSqlDatabaseTest {
+public class TestFileSizeCountTask extends AbstractReconSqlDBTest {
private FileCountBySizeDao fileCountBySizeDao;
@Before
- public void setUp() throws SQLException {
- UtilizationSchemaDefinition schemaDefinition =
- getInjector().getInstance(UtilizationSchemaDefinition.class);
- schemaDefinition.initializeSchema();
- Configuration sqlConfiguration =
- getInjector().getInstance((Configuration.class));
- fileCountBySizeDao = new FileCountBySizeDao(sqlConfiguration);
+ public void setUp() {
+ fileCountBySizeDao = getDao(FileCountBySizeDao.class);
}
@Test
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestReconTaskControllerImpl.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestReconTaskControllerImpl.java
index 81cc4e0..ad03e67 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestReconTaskControllerImpl.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestReconTaskControllerImpl.java
@@ -33,12 +33,10 @@
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.om.OMMetadataManager;
-import org.apache.hadoop.ozone.recon.persistence.AbstractSqlDatabaseTest;
+import org.apache.hadoop.ozone.recon.persistence.AbstractReconSqlDBTest;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
-import org.hadoop.ozone.recon.schema.ReconTaskSchemaDefinition;
import org.hadoop.ozone.recon.schema.tables.daos.ReconTaskStatusDao;
import org.hadoop.ozone.recon.schema.tables.pojos.ReconTaskStatus;
-import org.jooq.Configuration;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -46,32 +44,22 @@
/**
* Class used to test ReconTaskControllerImpl.
*/
-public class TestReconTaskControllerImpl extends AbstractSqlDatabaseTest {
+public class TestReconTaskControllerImpl extends AbstractReconSqlDBTest {
private ReconTaskController reconTaskController;
- private Configuration sqlConfiguration;
private ReconTaskStatusDao reconTaskStatusDao;
@Before
- public void setUp() throws Exception {
-
+ public void setUp() {
OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
-
- sqlConfiguration = getInjector()
- .getInstance(Configuration.class);
-
- reconTaskStatusDao = new ReconTaskStatusDao(sqlConfiguration);
- ReconTaskSchemaDefinition schemaDefinition = getInjector().
- getInstance(ReconTaskSchemaDefinition.class);
- schemaDefinition.initializeSchema();
-
+ reconTaskStatusDao = getDao(ReconTaskStatusDao.class);
reconTaskController = new ReconTaskControllerImpl(ozoneConfiguration,
reconTaskStatusDao, new HashSet<>());
reconTaskController.start();
}
@Test
- public void testRegisterTask() throws Exception {
+ public void testRegisterTask() {
String taskName = "Dummy_" + System.currentTimeMillis();
DummyReconDBTask dummyReconDBTask =
new DummyReconDBTask(taskName, DummyReconDBTask.TaskType.ALWAYS_PASS);
@@ -102,7 +90,7 @@
.process(any());
long endTime = System.currentTimeMillis();
- reconTaskStatusDao = new ReconTaskStatusDao(sqlConfiguration);
+ reconTaskStatusDao = getDao(ReconTaskStatusDao.class);
ReconTaskStatus reconTaskStatus = reconTaskStatusDao.findById("MockTask");
long taskTimeStamp = reconTaskStatus.getLastUpdatedTimestamp();
long seqNumber = reconTaskStatus.getLastUpdatedSeqNumber();
@@ -132,7 +120,7 @@
assertEquals(dummyReconDBTask, reconTaskController.getRegisteredTasks()
.get(dummyReconDBTask.getTaskName()));
- reconTaskStatusDao = new ReconTaskStatusDao(sqlConfiguration);
+ reconTaskStatusDao = getDao(ReconTaskStatusDao.class);
ReconTaskStatus dbRecord = reconTaskStatusDao.findById(taskName);
Assert.assertEquals(taskName, dbRecord.getTaskName());
@@ -168,7 +156,7 @@
omMetadataManagerMock);
assertTrue(reconTaskController.getRegisteredTasks().isEmpty());
- reconTaskStatusDao = new ReconTaskStatusDao(sqlConfiguration);
+ reconTaskStatusDao = getDao(ReconTaskStatusDao.class);
ReconTaskStatus dbRecord = reconTaskStatusDao.findById(taskName);
Assert.assertEquals(taskName, dbRecord.getTaskName());
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/types/GuiceInjectorUtilsForTests.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/types/GuiceInjectorUtilsForTests.java
deleted file mode 100644
index d147e58..0000000
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/types/GuiceInjectorUtilsForTests.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.recon.types;
-
-import com.google.inject.AbstractModule;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.Singleton;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.recon.persistence.AbstractSqlDatabaseTest;
-import org.apache.hadoop.ozone.recon.persistence.DataSourceConfiguration;
-import org.apache.hadoop.ozone.recon.persistence.JooqPersistenceModule;
-import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
-import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider;
-import org.apache.hadoop.ozone.recon.spi.OzoneManagerServiceProvider;
-import org.apache.hadoop.ozone.recon.spi.impl.ContainerDBServiceProviderImpl;
-import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl;
-import org.apache.hadoop.ozone.recon.spi.impl.ReconContainerDBProvider;
-import org.apache.hadoop.hdds.utils.db.DBStore;
-import org.hadoop.ozone.recon.schema.tables.daos.MissingContainersDao;
-import org.jooq.Configuration;
-import org.junit.Assert;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.File;
-import java.io.IOException;
-
-import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_DATANODE_ADDRESS_KEY;
-import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_DB_DIR;
-import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SCM_DB_DIR;
-import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_OM_SNAPSHOT_DB_DIR;
-
-/**
- * Utility methods to get guice injector and ozone configuration.
- */
-public interface GuiceInjectorUtilsForTests {
-
- /**
- * Get Guice Injector with bindings.
- * @param ozoneManagerServiceProvider
- * @param reconOMMetadataManager
- * @param temporaryFolder
- * @return Injector
- * @throws IOException ioEx.
- */
- default Injector getInjector(
- OzoneManagerServiceProviderImpl ozoneManagerServiceProvider,
- ReconOMMetadataManager reconOMMetadataManager,
- TemporaryFolder temporaryFolder
- ) throws IOException {
-
- File tempDir = temporaryFolder.newFolder();
- AbstractSqlDatabaseTest.DataSourceConfigurationProvider
- configurationProvider =
- new AbstractSqlDatabaseTest.DataSourceConfigurationProvider(tempDir);
-
- JooqPersistenceModule jooqPersistenceModule =
- new JooqPersistenceModule(configurationProvider);
-
- Injector baseInjector = Guice.createInjector(jooqPersistenceModule,
- new AbstractModule() {
- @Override
- protected void configure() {
- try {
- bind(DataSourceConfiguration.class)
- .toProvider(configurationProvider);
- bind(OzoneConfiguration.class).toInstance(
- getTestOzoneConfiguration(temporaryFolder));
-
- if (reconOMMetadataManager != null) {
- bind(ReconOMMetadataManager.class)
- .toInstance(reconOMMetadataManager);
- }
-
- if (ozoneManagerServiceProvider != null) {
- bind(OzoneManagerServiceProvider.class)
- .toInstance(ozoneManagerServiceProvider);
- }
-
- bind(DBStore.class).toProvider(ReconContainerDBProvider.class).
- in(Singleton.class);
- } catch (IOException e) {
- Assert.fail();
- }
- }
- });
-
- return baseInjector.createChildInjector(new AbstractModule() {
- @Override
- protected void configure() {
- Configuration sqlConfiguration =
- baseInjector.getInstance((Configuration.class));
- MissingContainersDao missingContainersDao =
- new MissingContainersDao(sqlConfiguration);
- bind(MissingContainersDao.class).toInstance(missingContainersDao);
- bind(ContainerDBServiceProvider.class).to(
- ContainerDBServiceProviderImpl.class).in(Singleton.class);
- }
- });
- }
-
- /**
- * Get Test OzoneConfiguration instance.
- * @return OzoneConfiguration
- * @throws IOException ioEx.
- */
- default OzoneConfiguration getTestOzoneConfiguration(
- TemporaryFolder temporaryFolder) throws IOException {
- OzoneConfiguration configuration = new OzoneConfiguration();
- configuration.set(OZONE_RECON_OM_SNAPSHOT_DB_DIR,
- temporaryFolder.newFolder().getAbsolutePath());
- configuration.set(OZONE_RECON_DB_DIR, temporaryFolder.newFolder()
- .getAbsolutePath());
- configuration.set(OZONE_RECON_SCM_DB_DIR, temporaryFolder.newFolder()
- .getAbsolutePath());
- configuration.set(OZONE_RECON_DATANODE_ADDRESS_KEY,
- "0.0.0.0:0");
- return configuration;
- }
-}
diff --git a/hadoop-ozone/s3gateway/pom.xml b/hadoop-ozone/s3gateway/pom.xml
index a52c6e7..80bd34f 100644
--- a/hadoop-ozone/s3gateway/pom.xml
+++ b/hadoop-ozone/s3gateway/pom.xml
@@ -38,56 +38,40 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-test-utils</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-server-framework</artifactId>
</dependency>
<dependency>
<groupId>org.jboss.weld.servlet</groupId>
<artifactId>weld-servlet</artifactId>
- <version>2.4.7.Final</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.containers</groupId>
<artifactId>jersey-container-servlet-core</artifactId>
- <version>2.27</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.ext.cdi</groupId>
<artifactId>jersey-cdi1x</artifactId>
- <version>2.27</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.inject</groupId>
<artifactId>jersey-hk2</artifactId>
- <version>2.27</version>
- <exclusions>
- <exclusion>
- <artifactId>hk2-api</artifactId>
- <groupId>org.glassfish.hk2</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hk2-utils</artifactId>
- <groupId>org.glassfish.hk2</groupId>
- </exclusion>
- <exclusion>
- <artifactId>aopalliance-repackaged</artifactId>
- <groupId>org.glassfish.hk2.external</groupId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.glassfish.hk2</groupId>
<artifactId>hk2-api</artifactId>
- <version>2.5.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-xml</artifactId>
- <version>2.9.0</version>
</dependency>
<dependency>
<groupId>javax.enterprise</groupId>
<artifactId>cdi-api</artifactId>
- <version>1.2</version>
</dependency>
<dependency>
<groupId>com.sun.xml.bind</groupId>
@@ -108,7 +92,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
+ <artifactId>hadoop-hdds-hadoop-dependency-server</artifactId>
<exclusions>
<exclusion>
<groupId>com.sun.jersey</groupId>
@@ -130,30 +114,7 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <exclusions>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-servlet</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <type>test-jar</type>
+ <artifactId>hadoop-hdds-hadoop-dependency-test</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java
index 09fcb01..05208bc 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java
@@ -22,7 +22,9 @@
import org.apache.hadoop.hdds.cli.GenericCli;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.tracing.TracingUtil;
+import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import picocli.CommandLine.Command;
@@ -47,8 +49,9 @@
@Override
public Void call() throws Exception {
OzoneConfiguration ozoneConfiguration = createOzoneConfiguration();
+ TracingUtil.initTracing("S3gateway", ozoneConfiguration);
OzoneConfigurationHolder.setConfiguration(ozoneConfiguration);
- ozoneConfiguration.set("hadoop.http.authentication.type", "simple");
+ UserGroupInformation.setConfiguration(ozoneConfiguration);
httpServer = new S3GatewayHttpServer(ozoneConfiguration, "s3gateway");
start();
return null;
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java
index 4ec22c5..3cd7b7c 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java
@@ -76,6 +76,9 @@
private OzoneClient getClient(OzoneConfiguration config) throws IOException {
try {
+ String awsAccessId = v4RequestParser.getAwsAccessId();
+ UserGroupInformation remoteUser =
+ UserGroupInformation.createRemoteUser(awsAccessId);
if (OzoneSecurityUtil.isSecurityEnabled(config)) {
LOG.debug("Creating s3 auth info for client.");
try {
@@ -84,8 +87,8 @@
identifier.setTokenType(S3AUTHINFO);
identifier.setStrToSign(v4RequestParser.getStringToSign());
identifier.setSignature(v4RequestParser.getSignature());
- identifier.setAwsAccessId(v4RequestParser.getAwsAccessId());
- identifier.setOwner(new Text(v4RequestParser.getAwsAccessId()));
+ identifier.setAwsAccessId(awsAccessId);
+ identifier.setOwner(new Text(awsAccessId));
if (LOG.isTraceEnabled()) {
LOG.trace("Adding token for service:{}", omService);
}
@@ -93,17 +96,14 @@
identifier.getSignature().getBytes(UTF_8),
identifier.getKind(),
omService);
- UserGroupInformation remoteUser =
- UserGroupInformation.createRemoteUser(
- v4RequestParser.getAwsAccessId());
remoteUser.addToken(token);
- UserGroupInformation.setLoginUser(remoteUser);
} catch (OS3Exception | URISyntaxException ex) {
LOG.error("S3 auth info creation failed.");
throw S3_AUTHINFO_CREATION_ERROR;
}
}
+ UserGroupInformation.setLoginUser(remoteUser);
} catch (Exception e) {
LOG.error("Error: ", e);
}
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java
index 2611f50..fae1c82 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java
@@ -37,15 +37,21 @@
"ozone.s3g.http-address";
public static final String OZONE_S3G_HTTPS_ADDRESS_KEY =
"ozone.s3g.https-address";
- public static final String OZONE_S3G_KEYTAB_FILE =
- "ozone.s3g.keytab.file";
+
public static final String OZONE_S3G_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
public static final int OZONE_S3G_HTTP_BIND_PORT_DEFAULT = 9878;
public static final int OZONE_S3G_HTTPS_BIND_PORT_DEFAULT = 9879;
- public static final String OZONE_S3G_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL =
- "ozone.s3g.authentication.kerberos.principal";
+
public static final String OZONE_S3G_DOMAIN_NAME = "ozone.s3g.domain.name";
+ public static final String OZONE_S3G_HTTP_AUTH_CONFIG_PREFIX =
+ "ozone.s3g.http.auth.";
+ public static final String OZONE_S3G_HTTP_AUTH_TYPE =
+ OZONE_S3G_HTTP_AUTH_CONFIG_PREFIX + "type";
+ public static final String OZONE_S3G_KEYTAB_FILE =
+ OZONE_S3G_HTTP_AUTH_CONFIG_PREFIX + "kerberos.keytab";
+ public static final String OZONE_S3G_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL =
+ OZONE_S3G_HTTP_AUTH_CONFIG_PREFIX + "kerberos.principal";
/**
* Never constructed.
*/
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayHttpServer.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayHttpServer.java
index b164e54..d8c0bf0 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayHttpServer.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayHttpServer.java
@@ -19,7 +19,7 @@
import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.server.http.BaseHttpServer;
/**
@@ -32,7 +32,7 @@
*/
public static final int FILTER_PRIORITY_DO_AFTER = 50;
- public S3GatewayHttpServer(Configuration conf,
+ public S3GatewayHttpServer(ConfigurationSource conf,
String name) throws IOException {
super(conf, name);
}
@@ -87,4 +87,14 @@
return S3GatewayConfigKeys.OZONE_S3G_HTTP_ENABLED_KEY;
}
+ @Override
+ protected String getHttpAuthType() {
+ return S3GatewayConfigKeys.OZONE_S3G_HTTP_AUTH_TYPE;
+ }
+
+ @Override
+ protected String getHttpAuthConfigPrefix() {
+ return S3GatewayConfigKeys.OZONE_S3G_HTTP_AUTH_CONFIG_PREFIX;
+ }
+
}
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/TracingFilter.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/TracingFilter.java
new file mode 100644
index 0000000..28e5665
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/TracingFilter.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.s3;
+
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.ContainerResponseContext;
+import javax.ws.rs.container.ContainerResponseFilter;
+import javax.ws.rs.container.ResourceInfo;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.ext.Provider;
+
+import io.opentracing.Scope;
+import io.opentracing.ScopeManager;
+import io.opentracing.Span;
+import io.opentracing.util.GlobalTracer;
+
+/**
+ * Filter used to add jaeger tracing span.
+ */
+
+@Provider
+public class TracingFilter implements ContainerRequestFilter,
+ ContainerResponseFilter {
+
+ public static final String TRACING_SCOPE = "TRACING_SCOPE";
+ public static final String TRACING_SPAN = "TRACING_SPAN";
+
+ @Context
+ private ResourceInfo resourceInfo;
+
+
+ @Override
+ public void filter(ContainerRequestContext requestContext) {
+ finishAndCloseActiveSpan();
+
+ Span span = GlobalTracer.get().buildSpan(
+ resourceInfo.getResourceClass().getSimpleName() + "." +
+ resourceInfo.getResourceMethod().getName()).start();
+ Scope scope = GlobalTracer.get().activateSpan(span);
+ requestContext.setProperty(TRACING_SCOPE, scope);
+ requestContext.setProperty(TRACING_SPAN, span);
+ }
+
+ @Override
+ public void filter(ContainerRequestContext requestContext,
+ ContainerResponseContext responseContext) {
+ Scope scope = (Scope)requestContext.getProperty(TRACING_SCOPE);
+ if (scope != null) {
+ scope.close();
+ }
+ Span span = (Span) requestContext.getProperty(TRACING_SPAN);
+ if (span != null) {
+ span.finish();
+ }
+
+ finishAndCloseActiveSpan();
+ }
+
+ private void finishAndCloseActiveSpan() {
+ ScopeManager scopeManager = GlobalTracer.get().scopeManager();
+ if (scopeManager != null && scopeManager.activeSpan() != null) {
+ scopeManager.activeSpan().finish();
+ scopeManager.activate(null);
+ }
+ }
+}
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
index 94c041b..ef02510 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
@@ -53,7 +53,6 @@
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import org.apache.commons.lang3.StringUtils;
-import static org.apache.hadoop.ozone.s3.util.OzoneS3Util.getS3Username;
import static org.apache.hadoop.ozone.s3.util.S3Consts.ENCODING_TYPE;
import org.apache.http.HttpStatus;
import org.slf4j.Logger;
@@ -202,10 +201,8 @@
public Response put(@PathParam("bucket") String bucketName, @Context
HttpHeaders httpHeaders) throws IOException, OS3Exception {
- String volumeName = getS3Username(getSignatureProcessor().getAwsAccessId());
-
try {
- String location = createS3Bucket(volumeName, bucketName);
+ String location = createS3Bucket(bucketName);
LOG.info("Location is {}", location);
return Response.status(HttpStatus.SC_OK).header("Location", location)
.build();
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/CompleteMultipartUploadRequest.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/CompleteMultipartUploadRequest.java
index 6120ad6..7228947 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/CompleteMultipartUploadRequest.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/CompleteMultipartUploadRequest.java
@@ -30,7 +30,8 @@
* Request for Complete Multipart Upload request.
*/
@XmlAccessorType(XmlAccessType.FIELD)
-@XmlRootElement(name = "CompleteMultipartUpload")
+@XmlRootElement(name = "CompleteMultipartUpload", namespace =
+ "http://s3.amazonaws.com/doc/2006-03-01/")
public class CompleteMultipartUploadRequest {
@XmlElement(name = "Part")
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/CompleteMultipartUploadRequestUnmarshaller.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/CompleteMultipartUploadRequestUnmarshaller.java
new file mode 100644
index 0000000..4b34c96
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/CompleteMultipartUploadRequestUnmarshaller.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.s3.endpoint;
+
+import org.xml.sax.InputSource;
+import org.xml.sax.XMLReader;
+
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.ext.MessageBodyReader;
+import javax.xml.XMLConstants;
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.UnmarshallerHandler;
+import javax.xml.parsers.SAXParserFactory;
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Type;
+
+import static org.apache.hadoop.ozone.s3.util.S3Consts.S3_XML_NAMESPACE;
+
+/**
+ * Custom unmarshaller to read CompleteMultipartUploadRequest wo namespace.
+ */
+
+public class CompleteMultipartUploadRequestUnmarshaller
+ implements MessageBodyReader<CompleteMultipartUploadRequest> {
+
+ private final JAXBContext context;
+ private final XMLReader xmlReader;
+
+ public CompleteMultipartUploadRequestUnmarshaller() {
+ try {
+ context = JAXBContext.newInstance(CompleteMultipartUploadRequest.class);
+ SAXParserFactory saxParserFactory = SAXParserFactory.newInstance();
+ saxParserFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+ xmlReader = saxParserFactory.newSAXParser().getXMLReader();
+ } catch (Exception ex) {
+ throw new AssertionError("Can not instantiate " +
+ "CompleteMultipartUploadRequest parser", ex);
+ }
+ }
+ @Override
+ public boolean isReadable(Class<?> aClass, Type type,
+ Annotation[] annotations, MediaType mediaType) {
+ return type.equals(CompleteMultipartUploadRequest.class);
+ }
+
+ @Override
+ public CompleteMultipartUploadRequest readFrom(
+ Class<CompleteMultipartUploadRequest> aClass, Type type,
+ Annotation[] annotations, MediaType mediaType,
+ MultivaluedMap<String, String> multivaluedMap,
+ InputStream inputStream) throws IOException, WebApplicationException {
+ try {
+ UnmarshallerHandler unmarshallerHandler =
+ context.createUnmarshaller().getUnmarshallerHandler();
+ XmlNamespaceFilter filter =
+ new XmlNamespaceFilter(S3_XML_NAMESPACE);
+ filter.setContentHandler(unmarshallerHandler);
+ filter.setParent(xmlReader);
+ filter.parse(new InputSource(inputStream));
+ return (CompleteMultipartUploadRequest) unmarshallerHandler.getResult();
+ } catch (Exception e) {
+ throw new WebApplicationException("Can't parse request body to XML.", e);
+ }
+ }
+}
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
index b7b4e69..b60519d 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
@@ -18,10 +18,12 @@
package org.apache.hadoop.ozone.s3.endpoint;
import javax.inject.Inject;
-import javax.ws.rs.NotFoundException;
import java.io.IOException;
+import java.util.Collections;
import java.util.Iterator;
+import java.util.function.Function;
+import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneVolume;
@@ -32,28 +34,18 @@
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
import com.google.common.annotations.VisibleForTesting;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* Basic helpers for all the REST endpoints.
*/
public class EndpointBase {
- private static final Logger LOG =
- LoggerFactory.getLogger(EndpointBase.class);
-
@Inject
private OzoneClient client;
@Inject
private SignatureProcessor signatureProcessor;
- protected OzoneBucket getBucket(String volumeName, String bucketName)
- throws IOException {
- return getVolume(volumeName).getBucket(bucketName);
- }
-
protected OzoneBucket getBucket(OzoneVolume volume, String bucketName)
throws OS3Exception, IOException {
OzoneBucket bucket;
@@ -73,11 +65,10 @@
throws OS3Exception, IOException {
OzoneBucket bucket;
try {
- OzoneVolume volume = getVolume(getOzoneVolumeName(bucketName));
- bucket = volume.getBucket(bucketName);
+ bucket = client.getObjectStore().getS3Bucket(bucketName);
} catch (OMException ex) {
if (ex.getResult() == ResultCodes.BUCKET_NOT_FOUND
- || ex.getResult() == ResultCodes.S3_BUCKET_NOT_FOUND) {
+ || ex.getResult() == ResultCodes.VOLUME_NOT_FOUND) {
throw S3ErrorTable.newError(S3ErrorTable.NO_SUCH_BUCKET, bucketName);
} else {
throw ex;
@@ -86,45 +77,31 @@
return bucket;
}
- protected OzoneVolume getVolume(String volumeName) throws IOException {
- OzoneVolume volume = null;
- try {
- volume = client.getObjectStore().getVolume(volumeName);
- } catch (OMException ex) {
- if (ex.getResult() == ResultCodes.VOLUME_NOT_FOUND) {
- throw new NotFoundException("Volume " + volumeName + " is not found");
- } else {
- throw ex;
- }
- }
- return volume;
+ protected OzoneVolume getVolume() throws IOException {
+ String s3VolumeName = HddsClientUtils.getS3VolumeName(
+ client.getConfiguration());
+ return client.getObjectStore().getVolume(s3VolumeName);
}
/**
* Create an S3Bucket, and also it creates mapping needed to access via
* ozone and S3.
- * @param userName
* @param bucketName
* @return location of the S3Bucket.
* @throws IOException
*/
- protected String createS3Bucket(String userName, String bucketName) throws
+ protected String createS3Bucket(String bucketName) throws
IOException {
try {
- client.getObjectStore().createS3Bucket(userName, bucketName);
+ client.getObjectStore().createS3Bucket(bucketName);
} catch (OMException ex) {
- if (ex.getResult() != ResultCodes.S3_BUCKET_ALREADY_EXISTS) {
+ if (ex.getResult() != ResultCodes.BUCKET_ALREADY_EXISTS) {
// S3 does not return error for bucket already exists, it just
// returns the location.
throw ex;
}
}
-
- // Not required to call as bucketname is same, but calling now in future
- // if mapping changes we get right location.
- String location = client.getObjectStore().getOzoneBucketName(
- bucketName);
- return "/"+location;
+ return "/" + bucketName;
}
/**
@@ -138,49 +115,16 @@
}
/**
- * Returns the Ozone Namespace for the S3Bucket. It will return the
- * OzoneVolume/OzoneBucketName.
- * @param s3BucketName - S3 Bucket Name.
- * @return String - The Ozone canonical name for this s3 bucket. This
- * string is useful for mounting an OzoneFS.
- * @throws IOException - Error is throw if the s3bucket does not exist.
- */
- public String getOzoneBucketMapping(String s3BucketName) throws IOException {
- return client.getObjectStore().getOzoneBucketMapping(s3BucketName);
- }
-
- /**
- * Returns the corresponding Ozone volume given an S3 Bucket.
- * @param s3BucketName - S3Bucket Name.
- * @return String - Ozone Volume name.
- * @throws IOException - Throws if the s3Bucket does not exist.
- */
- public String getOzoneVolumeName(String s3BucketName) throws IOException {
- return client.getObjectStore().getOzoneVolumeName(s3BucketName);
- }
-
- /**
- * Returns the corresponding Ozone bucket name for the given S3 bucket.
- * @param s3BucketName - S3Bucket Name.
- * @return String - Ozone bucket Name.
- * @throws IOException - Throws if the s3bucket does not exist.
- */
- public String getOzoneBucketName(String s3BucketName) throws IOException {
- return client.getObjectStore().getOzoneBucketName(s3BucketName);
- }
-
- /**
* Returns Iterator to iterate over all buckets for a specific user.
* The result can be restricted using bucket prefix, will return all
* buckets if bucket prefix is null.
*
- * @param userName
- * @param prefix
+ * @param prefix Bucket prefix to match
* @return {@code Iterator<OzoneBucket>}
*/
- public Iterator<? extends OzoneBucket> listS3Buckets(String userName,
- String prefix) {
- return client.getObjectStore().listS3Buckets(userName, prefix);
+ public Iterator<? extends OzoneBucket> listS3Buckets(String prefix)
+ throws IOException {
+ return iterateBuckets(volume -> volume.listBuckets(prefix));
}
/**
@@ -193,11 +137,23 @@
* @param previousBucket Buckets are listed after this bucket
* @return {@code Iterator<OzoneBucket>}
*/
- public Iterator<? extends OzoneBucket> listS3Buckets(String userName,
- String prefix,
- String previousBucket) {
- return client.getObjectStore().listS3Buckets(userName, prefix,
- previousBucket);
+ public Iterator<? extends OzoneBucket> listS3Buckets(String prefix,
+ String previousBucket) throws IOException {
+ return iterateBuckets(volume -> volume.listBuckets(prefix, previousBucket));
+ }
+
+ private Iterator<? extends OzoneBucket> iterateBuckets(
+ Function<OzoneVolume, Iterator<? extends OzoneBucket>> query)
+ throws IOException {
+ try {
+ return query.apply(getVolume());
+ } catch (OMException e) {
+ if (e.getResult() == ResultCodes.VOLUME_NOT_FOUND) {
+ return Collections.emptyIterator();
+ } else {
+ throw e;
+ }
+ }
}
public SignatureProcessor getSignatureProcessor() {
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index 455e734..f695fcb 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -259,8 +259,7 @@
try (S3WrapperInputStream s3WrapperInputStream =
new S3WrapperInputStream(
key.getInputStream())) {
- IOUtils.copyLarge(s3WrapperInputStream, dest, startOffset,
- copyLength);
+ s3WrapperInputStream.copyLarge(dest, startOffset, copyLength);
}
};
responseBuilder = Response
@@ -534,10 +533,16 @@
if (range != null) {
RangeHeader rangeHeader =
RangeHeaderParserUtil.parseRangeHeader(range, 0);
- IOUtils.copyLarge(sourceObject, ozoneOutputStream,
- rangeHeader.getStartOffset(),
- rangeHeader.getEndOffset() - rangeHeader.getStartOffset());
+ long copyLength = rangeHeader.getEndOffset() -
+ rangeHeader.getStartOffset();
+
+ try (S3WrapperInputStream s3WrapperInputStream =
+ new S3WrapperInputStream(
+ sourceObject.getInputStream())) {
+ s3WrapperInputStream.copyLarge(ozoneOutputStream,
+ rangeHeader.getStartOffset(), copyLength);
+ }
} else {
IOUtils.copy(sourceObject, ozoneOutputStream);
}
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java
index ae33250..0826276 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java
@@ -24,15 +24,12 @@
import java.util.Iterator;
import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.s3.commontypes.BucketMetadata;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.hadoop.ozone.s3.util.OzoneS3Util.getS3Username;
-
/**
* Top level rest endpoint.
*/
@@ -51,12 +48,9 @@
@GET
public Response get()
throws OS3Exception, IOException {
- OzoneVolume volume;
ListBucketResponse response = new ListBucketResponse();
- String userName = getS3Username(getSignatureProcessor().getAwsAccessId());
- Iterator<? extends OzoneBucket> bucketIterator = listS3Buckets(userName,
- null);
+ Iterator<? extends OzoneBucket> bucketIterator = listS3Buckets(null);
while (bucketIterator.hasNext()) {
OzoneBucket next = bucketIterator.next();
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/BadRequestExceptionMapper.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/BadRequestExceptionMapper.java
new file mode 100644
index 0000000..4fdf845
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/BadRequestExceptionMapper.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.exception;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.BadRequestException;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.ExceptionMapper;
+
+/**
+ * Class that represents BadRequestException.
+ */
+public class BadRequestExceptionMapper implements
+ ExceptionMapper<BadRequestException> {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(BadRequestExceptionMapper.class);
+ @Override
+ public Response toResponse(BadRequestException exception) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Returning exception. ex: {}", exception.toString());
+ }
+
+ return Response.status(Response.Status.BAD_REQUEST)
+ .entity(exception.getMessage()).build();
+ }
+}
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/S3WrapperInputStream.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/S3WrapperInputStream.java
index 9efcc87..edf90ed 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/S3WrapperInputStream.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/S3WrapperInputStream.java
@@ -23,12 +23,14 @@
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
/**
* S3Wrapper Input Stream which encapsulates KeyInputStream from ozone.
*/
public class S3WrapperInputStream extends FSInputStream {
private final KeyInputStream inputStream;
+ private static final int DEFAULT_BUFFER_SIZE = 32 * 1024;
/**
* Constructs S3WrapperInputStream with KeyInputStream.
@@ -76,4 +78,33 @@
public boolean seekToNewSource(long targetPos) throws IOException {
return false;
}
+
+ /**
+ * Copies some or all bytes from a large (over 2GB) <code>InputStream</code>
+ * to an <code>OutputStream</code>, optionally skipping input bytes.
+ * <p>
+ * Copy the method from IOUtils of commons-io to reimplement skip by seek
+ * rather than read. The reason why IOUtils of commons-io implement skip
+ * by read can be found at
+ * <a href="https://issues.apache.org/jira/browse/IO-203">IO-203</a>.
+ * </p>
+ * <p>
+ * This method buffers the input internally, so there is no need to use a
+ * <code>BufferedInputStream</code>.
+ * </p>
+ * The buffer size is given by {@link #DEFAULT_BUFFER_SIZE}.
+ *
+ * @param output the <code>OutputStream</code> to write to
+ * @param inputOffset : number of bytes to skip from input before copying
+ * -ve values are ignored
+ * @param length : number of bytes to copy. -ve means all
+ * @return the number of bytes copied
+ * @throws NullPointerException if the input or output is null
+ * @throws IOException if an I/O error occurs
+ */
+ public long copyLarge(final OutputStream output, final long inputOffset,
+ final long length) throws IOException {
+ return inputStream.copyLarge(output, inputOffset, length,
+ new byte[DEFAULT_BUFFER_SIZE]);
+ }
}
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
index b81a745..9f9440b 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
@@ -52,4 +52,7 @@
//Error code 416 is Range Not Satisfiable
public static final int RANGE_NOT_SATISFIABLE = 416;
+ public static final String S3_XML_NAMESPACE = "http://s3.amazonaws" +
+ ".com/doc/2006-03-01/";
+
}
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java
index 4feaca6..b0b8d43 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java
@@ -23,16 +23,16 @@
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
-import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_ALREADY_EXISTS;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_EMPTY;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND;
-import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.S3_BUCKET_NOT_FOUND;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND;
/**
@@ -45,9 +45,8 @@
}
private Map<String, OzoneVolumeStub> volumes = new HashMap<>();
- private Map<String, String> bucketVolumeMap = new HashMap<>();
private Map<String, Boolean> bucketEmptyStatus = new HashMap<>();
- private Map<String, List<OzoneBucket>> userBuckets = new HashMap<>();
+ private static OzoneConfiguration conf = new OzoneConfiguration();
@Override
public void createVolume(String volumeName) throws IOException {
@@ -121,89 +120,26 @@
}
@Override
- public void createS3Bucket(String userName, String s3BucketName) throws
+ public void createS3Bucket(String s3BucketName) throws
IOException {
- String volumeName = "s3" + userName;
- if (bucketVolumeMap.get(s3BucketName) == null) {
- bucketVolumeMap.put(s3BucketName, volumeName + "/" + s3BucketName);
+ if (!bucketEmptyStatus.containsKey(s3BucketName)) {
+ String volumeName = HddsClientUtils.getS3VolumeName(conf);
bucketEmptyStatus.put(s3BucketName, true);
- createVolume(volumeName);
+ if (!volumes.containsKey(volumeName)) {
+ createVolume(volumeName);
+ }
volumes.get(volumeName).createBucket(s3BucketName);
} else {
throw new OMException("", BUCKET_ALREADY_EXISTS);
}
-
- if (userBuckets.get(userName) == null) {
- List<OzoneBucket> ozoneBuckets = new ArrayList<>();
- ozoneBuckets.add(volumes.get(volumeName).getBucket(s3BucketName));
- userBuckets.put(userName, ozoneBuckets);
- } else {
- userBuckets.get(userName).add(volumes.get(volumeName).getBucket(
- s3BucketName));
- }
- }
-
- public Iterator<? extends OzoneBucket> listS3Buckets(String userName,
- String bucketPrefix) {
- if (userBuckets.get(userName) == null) {
- return new ArrayList<OzoneBucket>().iterator();
- } else {
- return userBuckets.get(userName).parallelStream()
- .filter(ozoneBucket -> {
- if (bucketPrefix != null) {
- return ozoneBucket.getName().startsWith(bucketPrefix);
- } else {
- return true;
- }
- }).collect(Collectors.toList())
- .iterator();
- }
- }
-
- public Iterator<? extends OzoneBucket> listS3Buckets(String userName,
- String bucketPrefix,
- String prevBucket) {
-
- if (userBuckets.get(userName) == null) {
- return new ArrayList<OzoneBucket>().iterator();
- } else {
- //Sort buckets lexicographically
- userBuckets.get(userName).sort(
- (bucket1, bucket2) -> {
- int compare = bucket1.getName().compareTo(bucket2.getName());
- if (compare < 0) {
- return -1;
- } else if (compare == 0) {
- return 0;
- } else {
- return 1;
- }
- });
- return userBuckets.get(userName).stream()
- .filter(ozoneBucket -> {
- if (prevBucket != null) {
- return ozoneBucket.getName().compareTo(prevBucket) > 0;
- } else {
- return true;
- }
- })
- .filter(ozoneBucket -> {
- if (bucketPrefix != null) {
- return ozoneBucket.getName().startsWith(bucketPrefix);
- } else {
- return true;
- }
- }).collect(Collectors.toList())
- .iterator();
- }
}
@Override
public void deleteS3Bucket(String s3BucketName) throws
IOException {
- if (bucketVolumeMap.containsKey(s3BucketName)) {
+ if (bucketEmptyStatus.containsKey(s3BucketName)) {
if (bucketEmptyStatus.get(s3BucketName)) {
- bucketVolumeMap.remove(s3BucketName);
+ bucketEmptyStatus.remove(s3BucketName);
} else {
throw new OMException("", BUCKET_NOT_EMPTY);
}
@@ -212,33 +148,7 @@
}
}
- @Override
- public String getOzoneBucketMapping(String s3BucketName) throws IOException {
- if (bucketVolumeMap.get(s3BucketName) == null) {
- throw new OMException("", S3_BUCKET_NOT_FOUND);
- }
- return bucketVolumeMap.get(s3BucketName);
- }
-
- @Override
- @SuppressWarnings("StringSplitter")
- public String getOzoneVolumeName(String s3BucketName) throws IOException {
- if (bucketVolumeMap.get(s3BucketName) == null) {
- throw new OMException("", S3_BUCKET_NOT_FOUND);
- }
- return bucketVolumeMap.get(s3BucketName).split("/")[0];
- }
-
- @Override
- @SuppressWarnings("StringSplitter")
- public String getOzoneBucketName(String s3BucketName) throws IOException {
- if (bucketVolumeMap.get(s3BucketName) == null) {
- throw new OMException("", BUCKET_NOT_FOUND);
- }
- return bucketVolumeMap.get(s3BucketName).split("/")[1];
- }
-
public void setBucketEmptyStatus(String bucketName, boolean status) {
- bucketEmptyStatus.put(bucketName, status);
+ bucketEmptyStatus.computeIfPresent(bucketName, (k, v) -> status);
}
}
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneClientStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneClientStub.java
index 3c7a253..e8237ca 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneClientStub.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneClientStub.java
@@ -19,8 +19,6 @@
*/
package org.apache.hadoop.ozone.client;
-import java.io.IOException;
-
/**
* In-memory OzoneClient for testing.
*/
@@ -31,7 +29,8 @@
}
@Override
- public void close() throws IOException {
+ public void close() {
//NOOP.
}
+
}
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java
index 8997260..67aa68a 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java
@@ -28,6 +28,7 @@
import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.ozone.OzoneAcl;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
/**
* Ozone volume with in-memory state for testing.
@@ -43,7 +44,7 @@
}
@Override
- public void createBucket(String bucketName) throws IOException {
+ public void createBucket(String bucketName) {
createBucket(bucketName, new BucketArgs.Builder()
.setStorageType(StorageType.DEFAULT)
.setVersioning(false)
@@ -51,8 +52,7 @@
}
@Override
- public void createBucket(String bucketName, BucketArgs bucketArgs)
- throws IOException {
+ public void createBucket(String bucketName, BucketArgs bucketArgs) {
buckets.put(bucketName, new OzoneBucketStub(
getName(),
bucketName,
@@ -67,7 +67,7 @@
if (buckets.containsKey(bucketName)) {
return buckets.get(bucketName);
} else {
- throw new IOException("BUCKET_NOT_FOUND");
+ throw new OMException("", OMException.ResultCodes.BUCKET_NOT_FOUND);
}
}
@@ -103,7 +103,7 @@
if (buckets.containsKey(bucketName)) {
buckets.remove(bucketName);
} else {
- throw new IOException("BUCKET_NOT_FOUND");
+ throw new OMException("", OMException.ResultCodes.BUCKET_NOT_FOUND);
}
}
}
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java
index 5a86514..e23fa60 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java
@@ -20,6 +20,7 @@
package org.apache.hadoop.ozone.s3.endpoint;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
@@ -46,8 +47,8 @@
String bucket = OzoneConsts.S3_BUCKET;
String key = OzoneConsts.KEY;
- OzoneClientStub client = new OzoneClientStub();
- client.getObjectStore().createS3Bucket(OzoneConsts.OZONE, bucket);
+ OzoneClient client = new OzoneClientStub();
+ client.getObjectStore().createS3Bucket(bucket);
HttpHeaders headers = Mockito.mock(HttpHeaders.class);
when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn(
@@ -59,7 +60,7 @@
Response response = rest.initializeMultipartUpload(bucket, key);
- assertEquals(response.getStatus(), 200);
+ assertEquals(200, response.getStatus());
MultipartUploadInitiateResponse multipartUploadInitiateResponse =
(MultipartUploadInitiateResponse) response.getEntity();
assertNotNull(multipartUploadInitiateResponse.getUploadID());
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketDelete.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketDelete.java
index 580a465..1df9583 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketDelete.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketDelete.java
@@ -25,6 +25,7 @@
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.ObjectStoreStub;
+import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
@@ -41,7 +42,7 @@
public class TestBucketDelete {
private String bucketName = OzoneConsts.BUCKET;
- private OzoneClientStub clientStub;
+ private OzoneClient clientStub;
private ObjectStore objectStoreStub;
private BucketEndpoint bucketEndpoint;
@@ -52,7 +53,7 @@
clientStub = new OzoneClientStub();
objectStoreStub = clientStub.getObjectStore();
- objectStoreStub.createS3Bucket(OzoneConsts.OZONE, bucketName);
+ clientStub.getObjectStore().createS3Bucket(bucketName);
// Create HeadBucket and setClient to OzoneClientStub
bucketEndpoint = new BucketEndpoint();
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketGet.java
index 844f9be..8e87000 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketGet.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketGet.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -187,7 +187,7 @@
"", null, null, null, null, null).getEntity();
Assert.assertTrue(getBucketResponse.isTruncated());
- Assert.assertTrue(getBucketResponse.getContents().size() == 2);
+ Assert.assertEquals(2, getBucketResponse.getContents().size());
// 2nd time
String continueToken = getBucketResponse.getNextToken();
@@ -195,7 +195,7 @@
(ListObjectResponse) getBucket.list("b1", null, null, null, maxKeys,
"", null, continueToken, null, null, null).getEntity();
Assert.assertTrue(getBucketResponse.isTruncated());
- Assert.assertTrue(getBucketResponse.getContents().size() == 2);
+ Assert.assertEquals(2, getBucketResponse.getContents().size());
continueToken = getBucketResponse.getNextToken();
@@ -206,7 +206,7 @@
"", null, continueToken, null, null, null).getEntity();
Assert.assertFalse(getBucketResponse.isTruncated());
- Assert.assertTrue(getBucketResponse.getContents().size() == 1);
+ Assert.assertEquals(1, getBucketResponse.getContents().size());
}
@@ -257,11 +257,11 @@
}
- @Test
/**
* This test is with prefix and delimiter and verify continuation-token
* behavior.
*/
+ @Test
public void listWithContinuationToken1() throws OS3Exception, IOException {
BucketEndpoint getBucket = new BucketEndpoint();
@@ -281,7 +281,7 @@
"dir", null, null, null, null, null).getEntity();
Assert.assertTrue(getBucketResponse.isTruncated());
- Assert.assertTrue(getBucketResponse.getCommonPrefixes().size() == 2);
+ Assert.assertEquals(2, getBucketResponse.getCommonPrefixes().size());
// 2nd time
String continueToken = getBucketResponse.getNextToken();
@@ -289,7 +289,7 @@
(ListObjectResponse) getBucket.list("b1", "/", null, null, maxKeys,
"dir", null, continueToken, null, null, null).getEntity();
Assert.assertTrue(getBucketResponse.isTruncated());
- Assert.assertTrue(getBucketResponse.getCommonPrefixes().size() == 2);
+ Assert.assertEquals(2, getBucketResponse.getCommonPrefixes().size());
//3rd time
continueToken = getBucketResponse.getNextToken();
@@ -298,12 +298,12 @@
"dir", null, continueToken, null, null, null).getEntity();
Assert.assertFalse(getBucketResponse.isTruncated());
- Assert.assertTrue(getBucketResponse.getCommonPrefixes().size() == 1);
+ Assert.assertEquals(1, getBucketResponse.getCommonPrefixes().size());
}
@Test
- public void listWithContinuationTokenFail() throws OS3Exception, IOException {
+ public void listWithContinuationTokenFail() throws IOException {
BucketEndpoint getBucket = new BucketEndpoint();
@@ -341,7 +341,7 @@
null, null, null, null, null, null).getEntity();
Assert.assertFalse(getBucketResponse.isTruncated());
- Assert.assertTrue(getBucketResponse.getContents().size() == 5);
+ Assert.assertEquals(5, getBucketResponse.getContents().size());
//As our list output is sorted, after seeking to startAfter, we shall
// have 4 keys.
@@ -352,14 +352,14 @@
1000, null, null, null, startAfter, null, null).getEntity();
Assert.assertFalse(getBucketResponse.isTruncated());
- Assert.assertTrue(getBucketResponse.getContents().size() == 4);
+ Assert.assertEquals(4, getBucketResponse.getContents().size());
getBucketResponse =
(ListObjectResponse) getBucket.list("b1", null, null, null,
1000, null, null, null, "random", null, null).getEntity();
Assert.assertFalse(getBucketResponse.isTruncated());
- Assert.assertTrue(getBucketResponse.getContents().size() == 0);
+ Assert.assertEquals(0, getBucketResponse.getContents().size());
}
@@ -367,14 +367,11 @@
private OzoneClient createClientWithKeys(String... keys) throws IOException {
OzoneClient client = new OzoneClientStub();
- client.getObjectStore().createS3Bucket("bilbo", "b1");
- String volume = client.getObjectStore().getOzoneVolumeName("b1");
- client.getObjectStore().getVolume(volume).createBucket("b1");
- OzoneBucket bucket =
- client.getObjectStore().getVolume(volume).getBucket("b1");
+ client.getObjectStore().createS3Bucket("b1");
+ OzoneBucket bucket = client.getObjectStore().getS3Bucket("b1");
for (String key : keys) {
bucket.createKey(key, 0).close();
}
return client;
}
-}
\ No newline at end of file
+}
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketHead.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketHead.java
index d9360ba..6f991e6 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketHead.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketHead.java
@@ -23,7 +23,7 @@
import javax.ws.rs.core.Response;
import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
import org.junit.Assert;
@@ -37,19 +37,13 @@
public class TestBucketHead {
private String bucketName = OzoneConsts.BUCKET;
- private String userName = OzoneConsts.OZONE;
- private OzoneClientStub clientStub;
- private ObjectStore objectStoreStub;
+ private OzoneClient clientStub;
private BucketEndpoint bucketEndpoint;
@Before
public void setup() throws Exception {
-
- //Create client stub and object store stub.
clientStub = new OzoneClientStub();
- objectStoreStub = clientStub.getObjectStore();
-
- objectStoreStub.createS3Bucket(userName, bucketName);
+ clientStub.getObjectStore().createS3Bucket(bucketName);
// Create HeadBucket and setClient to OzoneClientStub
bucketEndpoint = new BucketEndpoint();
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketPut.java
index 8ba8538..014cb3e 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketPut.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketPut.java
@@ -23,6 +23,7 @@
import javax.ws.rs.core.Response;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
import org.apache.hadoop.ozone.s3.SignatureProcessor;
@@ -46,7 +47,7 @@
public class TestBucketPut {
private String bucketName = OzoneConsts.BUCKET;
- private OzoneClientStub clientStub;
+ private OzoneClient clientStub;
private BucketEndpoint bucketEndpoint;
@Before
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestCompleteMultipartUploadRequestUnmarshaller.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestCompleteMultipartUploadRequestUnmarshaller.java
new file mode 100644
index 0000000..4a44a03
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestCompleteMultipartUploadRequestUnmarshaller.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.endpoint;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.List;
+import java.util.UUID;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+/**
+ * Class tests Unmarshall logic of {@link CompleteMultipartUploadRequest}.
+ */
+public class TestCompleteMultipartUploadRequestUnmarshaller {
+
+ private static String part1 = UUID.randomUUID().toString();
+ private static String part2 = UUID.randomUUID().toString();
+ @Test
+ public void fromStreamWithNamespace() throws IOException {
+ //GIVEN
+ ByteArrayInputStream inputBody =
+ new ByteArrayInputStream(
+ ("<CompleteMultipartUpload xmlns=\"http://s3.amazonaws" +
+ ".com/doc/2006-03-01/\">" +
+ "<Part><ETag>" + part1 + "</ETag><PartNumber>1" +
+ "</PartNumber></Part><Part><ETag>" + part2 +
+ "</ETag><PartNumber>2</PartNumber></Part>" +
+ "</CompleteMultipartUpload>")
+ .getBytes(UTF_8));
+
+ //WHEN
+ CompleteMultipartUploadRequest completeMultipartUploadRequest =
+ unmarshall(inputBody);
+
+ //THEN
+ checkContent(completeMultipartUploadRequest);
+ }
+
+ @Test
+ public void fromStreamWithoutNamespace() throws IOException {
+ //GIVEN
+ ByteArrayInputStream inputBody =
+ new ByteArrayInputStream(
+ ("<CompleteMultipartUpload>" +
+ "<Part><ETag>" + part1 + "</ETag><PartNumber>1</PartNumber" +
+ "></Part><Part><ETag>" + part2 + "</ETag><PartNumber>2" +
+ "</PartNumber></Part></CompleteMultipartUpload>")
+ .getBytes(UTF_8));
+
+ //WHEN
+ CompleteMultipartUploadRequest completeMultipartUploadRequest =
+ unmarshall(inputBody);
+
+ //THEN
+ checkContent(completeMultipartUploadRequest);
+ }
+
+ private void checkContent(CompleteMultipartUploadRequest request) {
+ Assert.assertEquals(2, request.getPartList().size());
+
+ List<CompleteMultipartUploadRequest.Part> parts =
+ request.getPartList();
+
+ Assert.assertEquals(part1, parts.get(0).geteTag());
+ Assert.assertEquals(part2, parts.get(1).geteTag());
+ }
+
+ private CompleteMultipartUploadRequest unmarshall(
+ ByteArrayInputStream inputBody) throws IOException {
+ return new CompleteMultipartUploadRequestUnmarshaller()
+ .readFrom(null, null, null, null, null, inputBody);
+ }
+}
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java
index f29e717..02d0f36 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java
@@ -21,9 +21,8 @@
package org.apache.hadoop.ozone.s3.endpoint;
import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
-import org.apache.hadoop.ozone.client.OzoneVolume;
import org.junit.Test;
import org.mockito.Mockito;
@@ -46,12 +45,8 @@
String bucket = OzoneConsts.S3_BUCKET;
String key = OzoneConsts.KEY;
- OzoneClientStub client = new OzoneClientStub();
- client.getObjectStore().createS3Bucket(OzoneConsts.OZONE, bucket);
- String volumeName = client.getObjectStore().getOzoneVolumeName(bucket);
- OzoneVolume volume = client.getObjectStore().getVolume(volumeName);
- OzoneBucket ozoneBucket = volume.getBucket(OzoneConsts.S3_BUCKET);
-
+ OzoneClient client = new OzoneClientStub();
+ client.getObjectStore().createS3Bucket(bucket);
HttpHeaders headers = Mockito.mock(HttpHeaders.class);
when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn(
@@ -63,7 +58,7 @@
Response response = rest.initializeMultipartUpload(bucket, key);
- assertEquals(response.getStatus(), 200);
+ assertEquals(200, response.getStatus());
MultipartUploadInitiateResponse multipartUploadInitiateResponse =
(MultipartUploadInitiateResponse) response.getEntity();
assertNotNull(multipartUploadInitiateResponse.getUploadID());
@@ -71,7 +66,7 @@
// Calling again should return different uploadID.
response = rest.initializeMultipartUpload(bucket, key);
- assertEquals(response.getStatus(), 200);
+ assertEquals(200, response.getStatus());
multipartUploadInitiateResponse =
(MultipartUploadInitiateResponse) response.getEntity();
assertNotNull(multipartUploadInitiateResponse.getUploadID());
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java
index 44cce58..337bfde 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.ozone.s3.endpoint;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
@@ -49,10 +50,8 @@
@BeforeClass
public static void setUp() throws Exception {
- OzoneClientStub client = new OzoneClientStub();
- client.getObjectStore().createS3Bucket(OzoneConsts.OZONE,
- OzoneConsts.S3_BUCKET);
-
+ OzoneClient client = new OzoneClientStub();
+ client.getObjectStore().createS3Bucket(OzoneConsts.S3_BUCKET);
HttpHeaders headers = Mockito.mock(HttpHeaders.class);
when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn(
@@ -68,7 +67,7 @@
assertNotNull(multipartUploadInitiateResponse.getUploadID());
uploadID = multipartUploadInitiateResponse.getUploadID();
- assertEquals(response.getStatus(), 200);
+ assertEquals(200, response.getStatus());
String content = "Multipart Upload";
ByteArrayInputStream body = new ByteArrayInputStream(content.getBytes());
@@ -97,7 +96,7 @@
(ListPartsResponse) response.getEntity();
Assert.assertFalse(listPartsResponse.getTruncated());
- Assert.assertTrue(listPartsResponse.getPartList().size() == 3);
+ assertEquals(3, listPartsResponse.getPartList().size());
}
@@ -109,7 +108,7 @@
(ListPartsResponse) response.getEntity();
Assert.assertTrue(listPartsResponse.getTruncated());
- Assert.assertTrue(listPartsResponse.getPartList().size() == 2);
+ assertEquals(2, listPartsResponse.getPartList().size());
// Continue
response = REST.get(OzoneConsts.S3_BUCKET, OzoneConsts.KEY, uploadID, 2,
@@ -117,7 +116,7 @@
listPartsResponse = (ListPartsResponse) response.getEntity();
Assert.assertFalse(listPartsResponse.getTruncated());
- Assert.assertTrue(listPartsResponse.getPartList().size() == 1);
+ assertEquals(1, listPartsResponse.getPartList().size());
}
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultiDeleteRequestUnmarshaller.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultiDeleteRequestUnmarshaller.java
index c15a128..b3ccfd7 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultiDeleteRequestUnmarshaller.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultiDeleteRequestUnmarshaller.java
@@ -25,7 +25,6 @@
import org.junit.Assert;
import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.junit.Assert.*;
import org.junit.Test;
/**
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java
index 2e6bf4e..80b5742 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java
@@ -21,6 +21,7 @@
package org.apache.hadoop.ozone.s3.endpoint;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
@@ -50,14 +51,13 @@
public class TestMultipartUploadComplete {
- private final static ObjectEndpoint REST = new ObjectEndpoint();;
- private final static OzoneClientStub CLIENT = new OzoneClientStub();
+ private final static ObjectEndpoint REST = new ObjectEndpoint();
+ private final static OzoneClient CLIENT = new OzoneClientStub();
@BeforeClass
public static void setUp() throws Exception {
- CLIENT.getObjectStore().createS3Bucket(OzoneConsts.OZONE,
- OzoneConsts.S3_BUCKET);
+ CLIENT.getObjectStore().createS3Bucket(OzoneConsts.S3_BUCKET);
HttpHeaders headers = Mockito.mock(HttpHeaders.class);
@@ -77,7 +77,7 @@
assertNotNull(multipartUploadInitiateResponse.getUploadID());
String uploadID = multipartUploadInitiateResponse.getUploadID();
- assertEquals(response.getStatus(), 200);
+ assertEquals(200, response.getStatus());
return uploadID;
@@ -88,7 +88,7 @@
ByteArrayInputStream body = new ByteArrayInputStream(content.getBytes());
Response response = REST.put(OzoneConsts.S3_BUCKET, key, content.length(),
partNumber, uploadID, body);
- assertEquals(response.getStatus(), 200);
+ assertEquals(200, response.getStatus());
assertNotNull(response.getHeaderString("ETag"));
Part part = new Part();
part.seteTag(response.getHeaderString("ETag"));
@@ -103,16 +103,16 @@
Response response = REST.completeMultipartUpload(OzoneConsts.S3_BUCKET, key,
uploadID, completeMultipartUploadRequest);
- assertEquals(response.getStatus(), 200);
+ assertEquals(200, response.getStatus());
CompleteMultipartUploadResponse completeMultipartUploadResponse =
(CompleteMultipartUploadResponse) response.getEntity();
- assertEquals(completeMultipartUploadResponse.getBucket(),
- OzoneConsts.S3_BUCKET);
- assertEquals(completeMultipartUploadResponse.getKey(), key);
- assertEquals(completeMultipartUploadResponse.getLocation(),
- OzoneConsts.S3_BUCKET);
+ assertEquals(OzoneConsts.S3_BUCKET,
+ completeMultipartUploadResponse.getBucket());
+ assertEquals(key, completeMultipartUploadResponse.getKey());
+ assertEquals(OzoneConsts.S3_BUCKET,
+ completeMultipartUploadResponse.getLocation());
assertNotNull(completeMultipartUploadResponse.getETag());
}
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java
index f688ff9..7adc62e 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java
@@ -35,8 +35,8 @@
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
import org.apache.hadoop.ozone.s3.endpoint.CompleteMultipartUploadRequest.Part;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
@@ -63,17 +63,16 @@
private final static String KEY = "key2";
private final static String EXISTING_KEY = "key1";
private static final String EXISTING_KEY_CONTENT = "testkey";
- private final static OzoneClientStub CLIENT = new OzoneClientStub();
+ private final static OzoneClient CLIENT = new OzoneClientStub();
private static final int RANGE_FROM = 2;
private static final int RANGE_TO = 4;
@BeforeClass
public static void setUp() throws Exception {
+ CLIENT.getObjectStore().createS3Bucket(OzoneConsts.S3_BUCKET);
- ObjectStore objectStore = CLIENT.getObjectStore();
- objectStore.createS3Bucket(OzoneConsts.OZONE, OzoneConsts.S3_BUCKET);
-
- OzoneBucket bucket = getOzoneBucket(objectStore, OzoneConsts.S3_BUCKET);
+ OzoneBucket bucket =
+ CLIENT.getObjectStore().getS3Bucket(OzoneConsts.S3_BUCKET);
byte[] keyContent = EXISTING_KEY_CONTENT.getBytes();
try (OutputStream stream = bucket
@@ -111,13 +110,6 @@
OzoneConsts.S3_BUCKET + "/" + EXISTING_KEY, null);
partsList.add(part2);
- partNumber = 3;
- Part part3 =
- uploadPartWithCopy(KEY, uploadID, partNumber,
- OzoneConsts.S3_BUCKET + "/" + EXISTING_KEY,
- "bytes=" + RANGE_FROM + "-" + RANGE_TO);
- partsList.add(part3);
-
// complete multipart upload
CompleteMultipartUploadRequest completeMultipartUploadRequest = new
CompleteMultipartUploadRequest();
@@ -126,12 +118,11 @@
completeMultipartUpload(KEY, completeMultipartUploadRequest,
uploadID);
- OzoneBucket bucket = getOzoneBucket(CLIENT.getObjectStore(),
- OzoneConsts.S3_BUCKET);
+ OzoneBucket bucket =
+ CLIENT.getObjectStore().getS3Bucket(OzoneConsts.S3_BUCKET);
try (InputStream is = bucket.readKey(KEY)) {
String keyContent = new Scanner(is).useDelimiter("\\A").next();
- Assert.assertEquals(content + EXISTING_KEY_CONTENT + EXISTING_KEY_CONTENT
- .substring(RANGE_FROM, RANGE_TO), keyContent);
+ Assert.assertEquals(content + EXISTING_KEY_CONTENT, keyContent);
}
}
@@ -145,7 +136,7 @@
assertNotNull(multipartUploadInitiateResponse.getUploadID());
String uploadID = multipartUploadInitiateResponse.getUploadID();
- assertEquals(response.getStatus(), 200);
+ assertEquals(200, response.getStatus());
return uploadID;
@@ -157,7 +148,7 @@
ByteArrayInputStream body = new ByteArrayInputStream(content.getBytes());
Response response = REST.put(OzoneConsts.S3_BUCKET, key, content.length(),
partNumber, uploadID, body);
- assertEquals(response.getStatus(), 200);
+ assertEquals(200, response.getStatus());
assertNotNull(response.getHeaderString("ETag"));
Part part = new Part();
part.seteTag(response.getHeaderString("ETag"));
@@ -179,7 +170,7 @@
ByteArrayInputStream body = new ByteArrayInputStream("".getBytes());
Response response = REST.put(OzoneConsts.S3_BUCKET, key, 0, partNumber,
uploadID, body);
- assertEquals(response.getStatus(), 200);
+ assertEquals(200, response.getStatus());
CopyPartResult result = (CopyPartResult) response.getEntity();
assertNotNull(result.getETag());
@@ -198,16 +189,16 @@
Response response = REST.completeMultipartUpload(OzoneConsts.S3_BUCKET, key,
uploadID, completeMultipartUploadRequest);
- assertEquals(response.getStatus(), 200);
+ assertEquals(200, response.getStatus());
CompleteMultipartUploadResponse completeMultipartUploadResponse =
(CompleteMultipartUploadResponse) response.getEntity();
- assertEquals(completeMultipartUploadResponse.getBucket(),
- OzoneConsts.S3_BUCKET);
- assertEquals(completeMultipartUploadResponse.getKey(), KEY);
- assertEquals(completeMultipartUploadResponse.getLocation(),
- OzoneConsts.S3_BUCKET);
+ assertEquals(OzoneConsts.S3_BUCKET,
+ completeMultipartUploadResponse.getBucket());
+ assertEquals(KEY, completeMultipartUploadResponse.getKey());
+ assertEquals(OzoneConsts.S3_BUCKET,
+ completeMultipartUploadResponse.getLocation());
assertNotNull(completeMultipartUploadResponse.getETag());
}
@@ -225,13 +216,4 @@
setHeaders(new HashMap<>());
}
- private static OzoneBucket getOzoneBucket(ObjectStore objectStore,
- String bucketName)
- throws IOException {
-
- String ozoneBucketName = objectStore.getOzoneBucketName(bucketName);
- String ozoneVolumeName = objectStore.getOzoneVolumeName(bucketName);
-
- return objectStore.getVolume(ozoneVolumeName).getBucket(ozoneBucketName);
- }
}
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectDelete.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectDelete.java
index b5d0c93..e85f598 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectDelete.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectDelete.java
@@ -38,12 +38,10 @@
public void delete() throws IOException, OS3Exception {
//GIVEN
OzoneClient client = new OzoneClientStub();
- client.getObjectStore().createS3Bucket("bilbo", "b1");
-
- String volumeName = client.getObjectStore().getOzoneVolumeName("b1");
+ client.getObjectStore().createS3Bucket("b1");
OzoneBucket bucket =
- client.getObjectStore().getVolume(volumeName).getBucket("b1");
+ client.getObjectStore().getS3Bucket("b1");
bucket.createKey("key1", 0).close();
@@ -57,4 +55,4 @@
Assert.assertFalse("Bucket Should not contain any key after delete",
bucket.listKeys("").hasNext());
}
-}
\ No newline at end of file
+}
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
index fcafe31..c807f10 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
@@ -24,12 +24,11 @@
import javax.ws.rs.core.Response;
import java.io.ByteArrayInputStream;
import java.io.IOException;
-import java.nio.charset.Charset;
import java.time.format.DateTimeFormatter;
import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
-import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
@@ -51,12 +50,9 @@
@Test
public void get() throws IOException, OS3Exception {
//GIVEN
- OzoneClientStub client = new OzoneClientStub();
- client.getObjectStore().createS3Bucket("bilbo", "b1");
- String volumeName = client.getObjectStore().getOzoneVolumeName("b1");
- OzoneVolume volume = client.getObjectStore().getVolume(volumeName);
- OzoneBucket bucket =
- volume.getBucket("b1");
+ OzoneClient client = new OzoneClientStub();
+ client.getObjectStore().createS3Bucket("b1");
+ OzoneBucket bucket = client.getObjectStore().getS3Bucket("b1");
OzoneOutputStream keyStream =
bucket.createKey("key1", CONTENT.getBytes(UTF_8).length);
keyStream.write(CONTENT.getBytes(UTF_8));
@@ -74,10 +70,10 @@
//THEN
OzoneInputStream ozoneInputStream =
- volume.getBucket("b1")
+ client.getObjectStore().getS3Bucket("b1")
.readKey("key1");
String keyContent =
- IOUtils.toString(ozoneInputStream, Charset.forName("UTF-8"));
+ IOUtils.toString(ozoneInputStream, UTF_8);
Assert.assertEquals(CONTENT, keyContent);
Assert.assertEquals("" + keyContent.length(),
@@ -87,4 +83,4 @@
.parse(response.getHeaderString("Last-Modified"));
}
-}
\ No newline at end of file
+}
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java
index ba39b28..a1b4fb1 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java
@@ -26,8 +26,8 @@
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
-import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
@@ -44,22 +44,18 @@
*/
public class TestObjectHead {
private String bucketName = "b1";
- private OzoneClientStub clientStub;
- private ObjectStore objectStoreStub;
private ObjectEndpoint keyEndpoint;
private OzoneBucket bucket;
@Before
public void setup() throws IOException {
//Create client stub and object store stub.
- clientStub = new OzoneClientStub();
- objectStoreStub = clientStub.getObjectStore();
+ OzoneClient clientStub = new OzoneClientStub();
// Create volume and bucket
- objectStoreStub.createS3Bucket("bilbo", bucketName);
- String volName = objectStoreStub.getOzoneVolumeName(bucketName);
+ clientStub.getObjectStore().createS3Bucket(bucketName);
- bucket = objectStoreStub.getVolume(volName).getBucket(bucketName);
+ bucket = clientStub.getObjectStore().getS3Bucket(bucketName);
// Create HeadBucket and setClient to OzoneClientStub
keyEndpoint = new ObjectEndpoint();
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java
index f4c3b94..b6089c8 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java
@@ -36,6 +36,8 @@
import org.junit.Assert;
import org.junit.Test;
+import static java.util.Collections.singleton;
+
/**
* Test object multi delete.
*/
@@ -96,21 +98,20 @@
.collect(Collectors.toSet());
//THEN
+ Assert.assertEquals(singleton("key3"), keysAtTheEnd);
Assert.assertEquals(0, response.getDeletedObjects().size());
Assert.assertEquals(0, response.getErrors().size());
}
private OzoneBucket initTestData(OzoneClient client) throws IOException {
- client.getObjectStore().createS3Bucket("bilbo", "b1");
-
- String volumeName = client.getObjectStore().getOzoneVolumeName("b1");
+ client.getObjectStore().createS3Bucket("b1");
OzoneBucket bucket =
- client.getObjectStore().getVolume(volumeName).getBucket("b1");
+ client.getObjectStore().getS3Bucket("b1");
bucket.createKey("key1", 0).close();
bucket.createKey("key2", 0).close();
bucket.createKey("key3", 0).close();
return bucket;
}
-}
\ No newline at end of file
+}
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
index 775a18b..83e3505 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
@@ -24,11 +24,10 @@
import javax.ws.rs.core.Response;
import java.io.ByteArrayInputStream;
import java.io.IOException;
-import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import org.apache.hadoop.hdds.client.ReplicationType;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
import org.apache.hadoop.ozone.client.OzoneKeyDetails;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
@@ -52,25 +51,22 @@
*/
public class TestObjectPut {
public static final String CONTENT = "0123456789";
- private String userName = OzoneConsts.OZONE;
private String bucketName = "b1";
private String keyName = "key1";
private String destBucket = "b2";
private String destkey = "key2";
private String nonexist = "nonexist";
- private OzoneClientStub clientStub;
- private ObjectStore objectStoreStub;
+ private OzoneClient clientStub;
private ObjectEndpoint objectEndpoint;
@Before
public void setup() throws IOException {
//Create client stub and object store stub.
clientStub = new OzoneClientStub();
- objectStoreStub = clientStub.getObjectStore();
// Create bucket
- objectStoreStub.createS3Bucket(userName, bucketName);
- objectStoreStub.createS3Bucket("ozone1", destBucket);
+ clientStub.getObjectStore().createS3Bucket(bucketName);
+ clientStub.getObjectStore().createS3Bucket(destBucket);
// Create PutObject and setClient to OzoneClientStub
objectEndpoint = new ObjectEndpoint();
@@ -90,13 +86,11 @@
//THEN
- String volumeName = clientStub.getObjectStore()
- .getOzoneVolumeName(bucketName);
OzoneInputStream ozoneInputStream =
- clientStub.getObjectStore().getVolume(volumeName).getBucket(bucketName)
+ clientStub.getObjectStore().getS3Bucket(bucketName)
.readKey(keyName);
String keyContent =
- IOUtils.toString(ozoneInputStream, Charset.forName("UTF-8"));
+ IOUtils.toString(ozoneInputStream, StandardCharsets.UTF_8);
Assert.assertEquals(200, response.getStatus());
Assert.assertEquals(CONTENT, keyContent);
@@ -122,13 +116,11 @@
new ByteArrayInputStream(chunkedContent.getBytes()));
//THEN
- String volumeName = clientStub.getObjectStore()
- .getOzoneVolumeName(bucketName);
OzoneInputStream ozoneInputStream =
- clientStub.getObjectStore().getVolume(volumeName).getBucket(bucketName)
+ clientStub.getObjectStore().getS3Bucket(bucketName)
.readKey(keyName);
String keyContent =
- IOUtils.toString(ozoneInputStream, Charset.forName("UTF-8"));
+ IOUtils.toString(ozoneInputStream, StandardCharsets.UTF_8);
Assert.assertEquals(200, response.getStatus());
Assert.assertEquals("1234567890abcde", keyContent);
@@ -145,14 +137,12 @@
Response response = objectEndpoint.put(bucketName, keyName,
CONTENT.length(), 1, null, body);
- String volumeName = clientStub.getObjectStore().getOzoneVolumeName(
- bucketName);
+ OzoneInputStream ozoneInputStream = clientStub.getObjectStore()
+ .getS3Bucket(bucketName)
+ .readKey(keyName);
- OzoneInputStream ozoneInputStream = clientStub.getObjectStore().getVolume(
- volumeName).getBucket(bucketName).readKey(keyName);
-
- String keyContent = IOUtils.toString(ozoneInputStream, Charset.forName(
- "UTF-8"));
+ String keyContent = IOUtils.toString(ozoneInputStream,
+ StandardCharsets.UTF_8);
Assert.assertEquals(200, response.getStatus());
Assert.assertEquals(CONTENT, keyContent);
@@ -166,11 +156,10 @@
null, body);
// Check destination key and response
- volumeName = clientStub.getObjectStore().getOzoneVolumeName(destBucket);
- ozoneInputStream = clientStub.getObjectStore().getVolume(volumeName)
- .getBucket(destBucket).readKey(destkey);
+ ozoneInputStream = clientStub.getObjectStore().getS3Bucket(destBucket)
+ .readKey(destkey);
- keyContent = IOUtils.toString(ozoneInputStream, Charset.forName("UTF-8"));
+ keyContent = IOUtils.toString(ozoneInputStream, StandardCharsets.UTF_8);
Assert.assertEquals(200, response.getStatus());
Assert.assertEquals(CONTENT, keyContent);
@@ -258,14 +247,11 @@
Response response = objectEndpoint.put(bucketName, keyName, CONTENT
.length(), 1, null, body);
- String volumeName = clientStub.getObjectStore()
- .getOzoneVolumeName(bucketName);
-
OzoneKeyDetails key =
- clientStub.getObjectStore().getVolume(volumeName).getBucket(bucketName)
+ clientStub.getObjectStore().getS3Bucket(bucketName)
.getKey(keyName);
//default type is set
Assert.assertEquals(ReplicationType.RATIS, key.getReplicationType());
}
-}
\ No newline at end of file
+}
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java
index 08db655..8834f9d 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java
@@ -21,6 +21,7 @@
package org.apache.hadoop.ozone.s3.endpoint;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.junit.BeforeClass;
@@ -50,9 +51,8 @@
@BeforeClass
public static void setUp() throws Exception {
- OzoneClientStub client = new OzoneClientStub();
- client.getObjectStore().createS3Bucket(OzoneConsts.OZONE,
- OzoneConsts.S3_BUCKET);
+ OzoneClient client = new OzoneClientStub();
+ client.getObjectStore().createS3Bucket(OzoneConsts.S3_BUCKET);
HttpHeaders headers = Mockito.mock(HttpHeaders.class);
@@ -74,7 +74,7 @@
assertNotNull(multipartUploadInitiateResponse.getUploadID());
String uploadID = multipartUploadInitiateResponse.getUploadID();
- assertEquals(response.getStatus(), 200);
+ assertEquals(200, response.getStatus());
String content = "Multipart Upload";
ByteArrayInputStream body = new ByteArrayInputStream(content.getBytes());
@@ -95,7 +95,7 @@
assertNotNull(multipartUploadInitiateResponse.getUploadID());
String uploadID = multipartUploadInitiateResponse.getUploadID();
- assertEquals(response.getStatus(), 200);
+ assertEquals(200, response.getStatus());
String content = "Multipart Upload";
ByteArrayInputStream body = new ByteArrayInputStream(content.getBytes());
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java
index d524481..02c3b7c 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java
@@ -21,13 +21,12 @@
package org.apache.hadoop.ozone.s3.endpoint;
import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
import static org.junit.Assert.assertEquals;
import org.apache.hadoop.ozone.s3.SignatureProcessor;
-import org.apache.hadoop.ozone.s3.util.OzoneS3Util;
import org.junit.Before;
import org.junit.Test;
@@ -36,17 +35,14 @@
*/
public class TestRootList {
- private OzoneClientStub clientStub;
- private ObjectStore objectStoreStub;
+ private OzoneClient clientStub;
private RootEndpoint rootEndpoint;
- private String userName = OzoneConsts.OZONE;
@Before
public void setup() throws Exception {
//Create client stub and object store stub.
clientStub = new OzoneClientStub();
- objectStoreStub = clientStub.getObjectStore();
// Create HeadBucket and setClient to OzoneClientStub
rootEndpoint = new RootEndpoint();
@@ -60,7 +56,7 @@
rootEndpoint.setSignatureProcessor(new SignatureProcessor() {
@Override
- public String getStringToSign() throws Exception {
+ public String getStringToSign() {
return null;
}
@@ -78,11 +74,10 @@
ListBucketResponse response =
(ListBucketResponse) rootEndpoint.get().getEntity();
assertEquals(0, response.getBucketsNum());
- String s3Username = OzoneS3Util.getS3Username(this.userName);
String bucketBaseName = "bucket-" + getClass().getName();
for(int i = 0; i < 10; i++) {
- objectStoreStub.createS3Bucket(s3Username, bucketBaseName + i);
+ clientStub.getObjectStore().createS3Bucket(bucketBaseName + i);
}
response = (ListBucketResponse) rootEndpoint.get().getEntity();
assertEquals(10, response.getBucketsNum());
diff --git a/hadoop-ozone/tools/pom.xml b/hadoop-ozone/tools/pom.xml
index c83851d..7fb0833 100644
--- a/hadoop-ozone/tools/pom.xml
+++ b/hadoop-ozone/tools/pom.xml
@@ -37,6 +37,11 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-common</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-test-utils</artifactId>
+ <scope>test</scope>
+ </dependency>
<!-- Genesis requires server side components -->
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -44,6 +49,10 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-tools</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-client</artifactId>
</dependency>
<dependency>
@@ -56,13 +65,11 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>compile</scope>
+ <artifactId>hadoop-hdfs</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <scope>compile</scope>
+ <groupId>org.apache.ratis</groupId>
+ <artifactId>ratis-tools</artifactId>
</dependency>
<dependency>
<groupId>com.sun.xml.bind</groupId>
@@ -79,12 +86,12 @@
<dependency>
<groupId>org.openjdk.jmh</groupId>
<artifactId>jmh-core</artifactId>
- <version>1.19</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.openjdk.jmh</groupId>
<artifactId>jmh-generator-annprocess</artifactId>
- <version>1.19</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>io.dropwizard.metrics</groupId>
@@ -92,8 +99,11 @@
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
- <version>1.11.615</version>
</dependency>
<dependency>
<groupId>com.github.spotbugs</groupId>
@@ -107,9 +117,8 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
+ <artifactId>hadoop-hdds-hadoop-dependency-test</artifactId>
<scope>test</scope>
- <type>test-jar</type>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/OzoneAdmin.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/OzoneAdmin.java
index 3df8a09..eebc388 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/OzoneAdmin.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/OzoneAdmin.java
@@ -19,10 +19,10 @@
import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.cli.GenericCli;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient;
@@ -105,7 +105,7 @@
}
}
- private void checkAndSetSCMAddressArg(Configuration conf) {
+ private void checkAndSetSCMAddressArg(ConfigurationSource conf) {
if (StringUtils.isNotEmpty(scm)) {
conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, scm);
}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ChunkDataNodeDetails.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ChunkDataNodeDetails.java
new file mode 100644
index 0000000..6019e58
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ChunkDataNodeDetails.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.debug;
+/**
+ * Class that gives datanode details on which the chunk is present.
+ */
+public class ChunkDataNodeDetails {
+ private String ipAddress;
+ private String hostName;
+
+ public ChunkDataNodeDetails(String ipAddress, String hostName) {
+ this.ipAddress = ipAddress;
+ this.hostName = hostName;
+ }
+
+ @Override
+ public String toString() {
+ return "{"
+ + "ipAddress='"
+ + ipAddress
+ + '\''
+ + ", hostName='"
+ + hostName
+ + '\''
+ + '}';
+ }
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ChunkDetails.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ChunkDetails.java
new file mode 100644
index 0000000..278c2bf
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ChunkDetails.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.debug;
+
+/**
+ * Class that gives chunkDetails.
+ */
+public class ChunkDetails {
+ private String chunkName;
+ private long chunkOffset;
+
+ public String getChunkName() {
+ return chunkName;
+ }
+
+ public void setChunkName(String chunkName) {
+ this.chunkName = chunkName;
+ }
+
+ @Override
+ public String toString() {
+ return "{"
+ + "chunkName='"
+ + chunkName
+ + '\''
+ + ", chunkOffset="
+ + chunkOffset
+ + '}';
+ }
+
+ public long getChunkOffset() {
+ return chunkOffset;
+ }
+
+ public void setChunkOffset(long chunkOffset) {
+ this.chunkOffset = chunkOffset;
+ }
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ChunkKeyHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ChunkKeyHandler.java
new file mode 100644
index 0000000..3d28d0a
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ChunkKeyHandler.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.debug;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.scm.XceiverClientManager;
+import org.apache.hadoop.hdds.scm.XceiverClientSpi;
+import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient;
+import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls;
+import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
+import org.apache.hadoop.hdds.tracing.TracingUtil;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.*;
+import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
+import org.apache.hadoop.ozone.container.common.impl.ChunkLayOutVersion;
+import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
+import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.ozone.shell.keys.KeyHandler;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.ratis.protocol.ClientId;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Parameters;
+
+/**
+ * Class that gives chunk location given a specific key.
+ */
+@Command(name = "chunkinfo",
+ description = "returns chunk location"
+ + " information about an existing key")
+public class ChunkKeyHandler extends KeyHandler {
+
+ @Parameters(arity = "1..1", description = "key to be located")
+ private String uri;
+
+ private ContainerOperationClient containerOperationClient;
+ private XceiverClientManager xceiverClientManager;
+ private XceiverClientSpi xceiverClient;
+ private final ClientId clientId = ClientId.randomId();
+ private OzoneManagerProtocol ozoneManagerClient;
+
+ private String getChunkLocationPath(String containerLocation) {
+ return containerLocation + File.separator + OzoneConsts.STORAGE_DIR_CHUNKS;
+ }
+
+ @Override
+ protected void execute(OzoneClient client, OzoneAddress address)
+ throws IOException, OzoneClientException {
+ containerOperationClient = new
+ ContainerOperationClient(createOzoneConfiguration());
+ xceiverClientManager = containerOperationClient
+ .getXceiverClientManager();
+ ozoneManagerClient = TracingUtil.createProxy(
+ new OzoneManagerProtocolClientSideTranslatorPB(
+ getConf(), clientId.toString(),
+ null, UserGroupInformation.getCurrentUser()),
+ OzoneManagerProtocol.class, getConf());
+ address.ensureKeyAddress();
+ JsonObject jsonObj = new JsonObject();
+ JsonElement element;
+ String volumeName = address.getVolumeName();
+ String bucketName = address.getBucketName();
+ String keyName = address.getKeyName();
+ List<ContainerProtos.ChunkInfo> tempchunks = null;
+ List<ChunkDetails> chunkDetailsList = new ArrayList<ChunkDetails>();
+ List<String> chunkPaths = new ArrayList<String>();
+ OmKeyArgs keyArgs = new OmKeyArgs.Builder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(keyName)
+ .setRefreshPipeline(true)
+ .build();
+ OmKeyInfo keyInfo = ozoneManagerClient.lookupKey(keyArgs);
+ List<OmKeyLocationInfo> locationInfos = keyInfo
+ .getLatestVersionLocations().getBlocksLatestVersionOnly();
+ // querying the keyLocations.The OM is queried to get containerID and
+ // localID pertaining to a given key
+ ChunkLayOutVersion chunkLayOutVersion = ChunkLayOutVersion
+ .getConfiguredVersion(getConf());
+ for (OmKeyLocationInfo keyLocation:locationInfos) {
+ ContainerChunkInfo containerChunkInfoVerbose = new ContainerChunkInfo();
+ ContainerChunkInfo containerChunkInfo = new ContainerChunkInfo();
+ long containerId = keyLocation.getContainerID();
+ Token<OzoneBlockTokenIdentifier> token = keyLocation.getToken();
+ xceiverClient = xceiverClientManager
+ .acquireClient(keyLocation.getPipeline());
+ // Datanode is queried to get chunk information.Thus querying the
+ // OM,SCM and datanode helps us get chunk location information
+ if (token != null) {
+ UserGroupInformation.getCurrentUser().addToken(token);
+ }
+ ContainerProtos.DatanodeBlockID datanodeBlockID = keyLocation.getBlockID()
+ .getDatanodeBlockIDProtobuf();
+ ContainerProtos.GetBlockResponseProto response = ContainerProtocolCalls
+ .getBlock(xceiverClient, datanodeBlockID);
+ tempchunks = response.getBlockData().getChunksList();
+ ContainerProtos.ContainerDataProto containerData =
+ containerOperationClient.readContainer(
+ keyLocation.getContainerID(),
+ keyLocation.getPipeline());
+ for (ContainerProtos.ChunkInfo chunkInfo:tempchunks) {
+ ChunkDetails chunkDetails = new ChunkDetails();
+ chunkDetails.setChunkName(chunkInfo.getChunkName());
+ chunkDetails.setChunkOffset(chunkInfo.getOffset());
+ chunkDetailsList.add(chunkDetails);
+ chunkPaths.add(chunkLayOutVersion.getChunkFile(new File(
+ getChunkLocationPath(containerData.getContainerPath())),
+ keyLocation.getBlockID(),
+ ChunkInfo.getFromProtoBuf(chunkInfo)).toString());
+ }
+ containerChunkInfoVerbose
+ .setContainerPath(containerData.getContainerPath());
+ containerChunkInfoVerbose
+ .setDataNodeList(keyLocation.getPipeline().getNodes());
+ containerChunkInfoVerbose.setPipeline(keyLocation.getPipeline());
+ containerChunkInfoVerbose.setChunkInfos(chunkDetailsList);
+ containerChunkInfo.setFiles(chunkPaths);
+ List<ChunkDataNodeDetails> chunkDataNodeDetails = new
+ ArrayList<ChunkDataNodeDetails>();
+ for (DatanodeDetails datanodeDetails:keyLocation
+ .getPipeline().getNodes()) {
+ chunkDataNodeDetails.add(
+ new ChunkDataNodeDetails(datanodeDetails.getIpAddress(),
+ datanodeDetails.getHostName()));
+ }
+ containerChunkInfo.setChunkDataNodeDetails(chunkDataNodeDetails);
+ containerChunkInfo.setPipelineID(
+ keyLocation.getPipeline().getId().getId());
+ Gson gson = new GsonBuilder().create();
+ if (isVerbose()) {
+ element = gson.toJsonTree(containerChunkInfoVerbose);
+ jsonObj.add("container Id :" + containerId + " "
+ + "blockId :" + keyLocation.getLocalID() + "", element);
+ } else {
+ element = gson.toJsonTree(containerChunkInfo);
+ jsonObj.add("container Id :" + containerId + " "
+ + "blockId :" + keyLocation.getLocalID() + "", element);
+ }
+ }
+ xceiverClientManager.releaseClient(xceiverClient, false);
+ Gson gson = new GsonBuilder().setPrettyPrinting().create();
+ String prettyJson = gson.toJson(jsonObj);
+ System.out.println(prettyJson);
+ }
+
+
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ContainerChunkInfo.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ContainerChunkInfo.java
new file mode 100644
index 0000000..0e969c7
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ContainerChunkInfo.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.debug;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import java.util.List;
+import java.util.UUID;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+
+/**
+ * Class that gives container and chunk Information.
+ */
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ContainerChunkInfo {
+ private String containerPath;
+ private List<DatanodeDetails> dataNodeList;
+ private List<ChunkDetails> chunkInfos;
+ private List<String> files;
+ private List<ChunkDataNodeDetails> chunkDataNodeDetails;
+ private UUID pipelineID;
+ private Pipeline pipeline;
+
+ public void setChunkDataNodeDetails(List<ChunkDataNodeDetails>
+ chunkDataNodeDetails) {
+ this.chunkDataNodeDetails = chunkDataNodeDetails;
+ }
+
+ public void setFiles(List<String> files) {
+ this.files = files;
+ }
+
+ public void setPipelineID(UUID pipelineID) {
+ this.pipelineID = pipelineID;
+ }
+
+ public Pipeline getPipeline() {
+ return pipeline;
+ }
+
+ public void setPipeline(Pipeline pipeline) {
+ this.pipeline = pipeline;
+ }
+
+ public void setContainerPath(String containerPath) {
+ this.containerPath = containerPath;
+ }
+
+ public void setChunkInfos(List<ChunkDetails> chunkInfos) {
+ this.chunkInfos = chunkInfos;
+ }
+
+ public void setDataNodeList(List<DatanodeDetails> dataNodeList) {
+ this.dataNodeList = dataNodeList;
+ }
+
+ @Override
+ public String toString() {
+ return "Container{"
+ + "containerPath='"
+ + containerPath
+ + '\''
+ + ", dataNodeList="
+ + dataNodeList
+ + ", chunkInfos="
+ + chunkInfos
+ + ", pipeline="
+ + pipeline
+ + '}'
+ + "files="
+ + files
+ + "chunkdatanodeDetails="
+ + chunkDataNodeDetails
+ + "PipelineID="
+ + pipelineID;
+ }
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/OzoneDebug.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/OzoneDebug.java
new file mode 100644
index 0000000..25295f7
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/OzoneDebug.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.debug;
+
+import org.apache.hadoop.hdds.cli.GenericCli;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.ozone.segmentparser.RatisLogParser;
+import picocli.CommandLine;
+
+/**
+ * Ozone Debug Command line tool.
+ */
+@CommandLine.Command(name = "ozone debug",
+ description = "Developer tools for Ozone Debug operations",
+ versionProvider = HddsVersionProvider.class,
+ subcommands = {
+ ChunkKeyHandler.class,
+ RatisLogParser.class
+ },
+ mixinStandardHelpOptions = true)
+public class OzoneDebug extends GenericCli {
+
+ /**
+ * Main for the Ozone Debug shell Command handling.
+ *
+ * @param argv - System Args Strings[]
+ * @throws Exception
+ */
+ public static void main(String[] argv) throws Exception {
+
+ new OzoneDebug().run(argv);
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/package-info.java
similarity index 91%
copy from hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/package-info.java
copy to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/package-info.java
index 80c1985..8077bf3 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/package-info.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/package-info.java
@@ -14,8 +14,11 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
+ * <p>
+ *
*/
-package org.apache.hadoop.ozone.web.ozShell;
+
/**
- * Tests for ozone shell..
+ * Ozone Debug tools.
*/
+package org.apache.hadoop.ozone.debug;
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java
index 6b73963..1d57136 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java
@@ -29,8 +29,6 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import com.codahale.metrics.ScheduledReporter;
-import com.codahale.metrics.Slf4jReporter;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
@@ -53,7 +51,10 @@
import com.codahale.metrics.ConsoleReporter;
import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.ScheduledReporter;
+import com.codahale.metrics.Slf4jReporter;
import io.opentracing.Scope;
+import io.opentracing.Span;
import io.opentracing.util.GlobalTracer;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.RandomStringUtils;
@@ -170,18 +171,16 @@
* @param taskId unique ID of the task
*/
private void tryNextTask(TaskProvider provider, long taskId) {
- Scope scope =
- GlobalTracer.get().buildSpan(spanName)
- .startActive(true);
- try {
+ Span span = GlobalTracer.get().buildSpan(spanName).start();
+ try (Scope scope = GlobalTracer.get().activateSpan(span)) {
provider.executeNextTask(taskId);
successCounter.incrementAndGet();
} catch (Exception e) {
- scope.span().setTag("failure", true);
+ span.setTag("failure", true);
failureCounter.incrementAndGet();
LOG.error("Error on executing task {}", taskId, e);
} finally {
- scope.close();
+ span.finish();
}
}
@@ -237,7 +236,7 @@
attemptCounter = new AtomicLong(0);
if (prefix.length() == 0) {
- prefix = RandomStringUtils.randomAlphanumeric(10);
+ prefix = RandomStringUtils.randomAlphanumeric(10).toLowerCase();
} else {
//replace environment variables to support multi-node execution
prefix = resolvePrefix(prefix);
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
index 6e63a75..cc78332 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
@@ -42,6 +42,7 @@
OmKeyGenerator.class,
OmBucketGenerator.class,
HadoopFsGenerator.class,
+ HadoopNestedDirGenerator.class,
HadoopFsValidator.class,
SameKeyReader.class,
S3KeyGenerator.class,
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/FreonHttpServer.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/FreonHttpServer.java
index c8deb98..d16a3f2 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/FreonHttpServer.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/FreonHttpServer.java
@@ -19,7 +19,7 @@
import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.server.http.BaseHttpServer;
import org.apache.hadoop.ozone.OzoneConfigKeys;
@@ -27,7 +27,7 @@
* Http server to provide metrics + profile endpoint.
*/
public class FreonHttpServer extends BaseHttpServer {
- public FreonHttpServer(Configuration conf) throws IOException {
+ public FreonHttpServer(ConfigurationSource conf) throws IOException {
super(conf, "freon");
}
@@ -71,4 +71,14 @@
@Override protected String getEnabledKey() {
return OzoneConfigKeys.OZONE_FREON_HTTP_ENABLED_KEY;
}
+
+ @Override
+ protected String getHttpAuthType() {
+ return OzoneConfigKeys.OZONE_FREON_HTTP_AUTH_TYPE;
+ }
+
+ @Override
+ protected String getHttpAuthConfigPrefix() {
+ return OzoneConfigKeys.OZONE_FREON_HTTP_AUTH_CONFIG_PREFIX;
+ }
}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopNestedDirGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopNestedDirGenerator.java
new file mode 100644
index 0000000..72d096c
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopNestedDirGenerator.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.freon;
+
+import java.net.URI;
+import java.util.concurrent.Callable;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+
+/**
+ * Directory Generator tool to test OM performance.
+ */
+@Command(name = "ddsg",
+ aliases = "dfs-directory-generator",
+ description =
+ "Create nested directories to the any dfs compatible file system.",
+ versionProvider = HddsVersionProvider.class,
+ mixinStandardHelpOptions = true,
+ showDefaultValues = true)
+public class HadoopNestedDirGenerator extends BaseFreonGenerator
+ implements Callable<Void> {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(HadoopNestedDirGenerator.class);
+
+ @Option(names = {"-r", "--rpath"},
+ description = "Hadoop FS directory system path",
+ defaultValue = "o3fs://bucket2.vol2")
+ private String rootPath;
+
+ @Option(names = {"-d", "--depth"},
+ description = "Number of directories to be generated recursively",
+ defaultValue = "5")
+ private int depth;
+
+ @Option(names = {"-s", "--span"},
+ description =
+ "Number of child directories to be created in leaf directory.",
+ defaultValue = "10")
+ private int span;
+
+ @Option(names = {"-l", "--nameLen"},
+ description =
+ "Length of the random name of directory you want to create.",
+ defaultValue = "10")
+ private int length;
+
+ private FileSystem fileSystem;
+
+ @Override
+ public Void call() throws Exception {
+
+ init();
+ OzoneConfiguration configuration = createOzoneConfiguration();
+ fileSystem = FileSystem.get(URI.create(rootPath), configuration);
+ runTests(this::createDir);
+ return null;
+
+ }
+
+ /*
+ Nested directories will be created like this,
+ suppose you pass depth=3, span=3 and number of tests=2
+
+ Dir11 Dir12
+ | |
+ Dir21 Dir22
+ | |
+ Dir31 Dir32
+ / | \ / | \
+ Dir311 Dir312 Dir313 Dir321 Dir322 Dir323
+
+ */
+ private void createDir(long counter) throws Exception {
+ String dirString = RandomStringUtils.randomAlphanumeric(length);
+ for (int i = 1; i <= depth; i++) {
+ dirString = dirString.concat("/").concat(RandomStringUtils.
+ randomAlphanumeric(length));
+ }
+ Path file = new Path(rootPath.concat("/").concat(dirString));
+ fileSystem.mkdirs(file.getParent());
+ String leafDir = dirString.substring(0, dirString.length() - length);
+ String tmp = "/0";
+ for (int i = 1; i <= span; i++) {
+ String childDir = leafDir.concat(Integer.toString(i)).concat(tmp);
+ Path dir = new Path(rootPath.concat("/").concat(childDir));
+ fileSystem.mkdirs(dir.getParent());
+ }
+ }
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmKeyGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmKeyGenerator.java
index a0a135b..8c37659 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmKeyGenerator.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmKeyGenerator.java
@@ -27,12 +27,16 @@
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs.Builder;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
+import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import com.codahale.metrics.Timer;
+import org.apache.hadoop.security.UserGroupInformation;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
+import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.ALL;
+
/**
* Data generator tool test om performance.
*/
@@ -99,7 +103,7 @@
}
private void createKey(long counter) throws Exception {
-
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
OmKeyArgs keyArgs = new Builder()
.setBucketName(bucketName)
.setVolumeName(volumeName)
@@ -107,6 +111,8 @@
.setFactor(factor)
.setKeyName(generateObjectName(counter))
.setLocationInfoList(new ArrayList<>())
+ .setAcls(OzoneAclUtil.getAclList(ugi.getUserName(), ugi.getGroups(),
+ ALL, ALL))
.build();
timer.time(() -> {
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
index 9e1a45c..4751672 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
@@ -22,6 +22,7 @@
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintStream;
+import java.security.MessageDigest;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.HashMap;
@@ -31,21 +32,19 @@
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.LongSupplier;
-import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
-import io.opentracing.Scope;
-import io.opentracing.util.GlobalTracer;
-import org.apache.commons.codec.digest.DigestUtils;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.client.OzoneQuota;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.tracing.TracingUtil;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.ObjectStore;
@@ -66,6 +65,8 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.google.common.annotations.VisibleForTesting;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.time.DurationFormatUtils;
import org.slf4j.Logger;
@@ -73,8 +74,6 @@
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
import picocli.CommandLine.ParentCommand;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.security.MessageDigest;
/**
* Data generator tool to generate as much keys as possible.
@@ -623,8 +622,8 @@
String volumeName = "vol-" + volumeNumber + "-"
+ RandomStringUtils.randomNumeric(5);
LOG.trace("Creating volume: {}", volumeName);
- try (Scope ignored = GlobalTracer.get().buildSpan("createVolume")
- .startActive(true)) {
+ try (AutoCloseable scope = TracingUtil
+ .createActivatedSpan("createVolume")) {
long start = System.nanoTime();
objectStore.createVolume(volumeName);
long volumeCreationDuration = System.nanoTime() - start;
@@ -655,8 +654,9 @@
RandomStringUtils.randomNumeric(5);
LOG.trace("Creating bucket: {} in volume: {}",
bucketName, volume.getName());
- try (Scope ignored = GlobalTracer.get().buildSpan("createBucket")
- .startActive(true)) {
+ try (AutoCloseable scope = TracingUtil
+ .createActivatedSpan("createBucket")) {
+
long start = System.nanoTime();
volume.createBucket(bucketName);
long bucketCreationDuration = System.nanoTime() - start;
@@ -691,8 +691,7 @@
LOG.trace("Adding key: {} in bucket: {} of volume: {}",
keyName, bucketName, volumeName);
try {
- try (Scope scope = GlobalTracer.get().buildSpan("createKey")
- .startActive(true)) {
+ try (AutoCloseable scope = TracingUtil.createActivatedSpan("createKey")) {
long keyCreateStart = System.nanoTime();
try (OzoneOutputStream os = bucket.createKey(keyName, keySize, type,
factor, new HashMap<>())) {
@@ -701,8 +700,8 @@
.update(keyCreationDuration);
keyCreationTime.getAndAdd(keyCreationDuration);
- try (Scope writeScope = GlobalTracer.get().buildSpan("writeKeyData")
- .startActive(true)) {
+ try (AutoCloseable writeScope = TracingUtil
+ .createActivatedSpan("writeKeyData")) {
long keyWriteStart = System.nanoTime();
for (long nrRemaining = keySize;
nrRemaining > 0; nrRemaining -= bufferSize) {
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkContainerStateMap.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkContainerStateMap.java
index 66a1c3c..63e4c6e 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkContainerStateMap.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkContainerStateMap.java
@@ -73,7 +73,7 @@
.setReplicationFactor(pipeline.getFactor())
.setUsedBytes(0)
.setNumberOfKeys(0)
- .setStateEnterTime(Time.monotonicNow())
+ .setStateEnterTime(Time.now())
.setOwner(OzoneConsts.OZONE)
.setContainerID(x)
.setDeleteTransactionId(0)
@@ -93,7 +93,7 @@
.setReplicationFactor(pipeline.getFactor())
.setUsedBytes(0)
.setNumberOfKeys(0)
- .setStateEnterTime(Time.monotonicNow())
+ .setStateEnterTime(Time.now())
.setOwner(OzoneConsts.OZONE)
.setContainerID(y)
.setDeleteTransactionId(0)
@@ -112,7 +112,7 @@
.setReplicationFactor(pipeline.getFactor())
.setUsedBytes(0)
.setNumberOfKeys(0)
- .setStateEnterTime(Time.monotonicNow())
+ .setStateEnterTime(Time.now())
.setOwner(OzoneConsts.OZONE)
.setContainerID(currentCount++)
.setDeleteTransactionId(0)
@@ -181,7 +181,7 @@
.setReplicationFactor(pipeline.getFactor())
.setUsedBytes(0)
.setNumberOfKeys(0)
- .setStateEnterTime(Time.monotonicNow())
+ .setStateEnterTime(Time.now())
.setOwner(OzoneConsts.OZONE)
.setContainerID(cid)
.setDeleteTransactionId(0)
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
index 50b0c8e..d7b3576 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
@@ -26,7 +26,6 @@
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -45,13 +44,13 @@
import org.apache.hadoop.ozone.container.common.interfaces.Handler;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.DatanodeStates;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import com.google.common.collect.Maps;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.Level;
@@ -92,7 +91,7 @@
// 1 MB of data
data = ByteString.copyFromUtf8(RandomStringUtils.randomAscii(CHUNK_SIZE));
random = new Random();
- Configuration conf = new OzoneConfiguration();
+ OzoneConfiguration conf = new OzoneConfiguration();
baseDir = System.getProperty("java.io.tmpdir") + File.separator +
datanodeUuid;
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkOzoneManager.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkOzoneManager.java
index 2e2b610..19b5e8e 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkOzoneManager.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkOzoneManager.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.ozone.genesis;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT;
-
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
@@ -27,7 +25,6 @@
import java.util.UUID;
import java.util.concurrent.locks.ReentrantLock;
-import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -47,7 +44,9 @@
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authentication.client.AuthenticationException;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.Level;
import org.openjdk.jmh.annotations.Scope;
@@ -77,7 +76,7 @@
@Setup(Level.Trial)
public static void initialize()
- throws IOException, AuthenticationException, InterruptedException {
+ throws Exception {
try {
lock.lock();
if (scm == null) {
@@ -104,7 +103,7 @@
pipelineManager.openPipeline(pipeline.getId());
}
scm.getEventQueue().fireEvent(SCMEvents.SAFE_MODE_STATUS,
- new SCMSafeModeManager.SafeModeStatus(false));
+ new SCMSafeModeManager.SafeModeStatus(false, false));
Thread.sleep(1000);
// prepare OM
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkSCM.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkSCM.java
index f8c74dd..64e2f4d 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkSCM.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkSCM.java
@@ -18,13 +18,10 @@
package org.apache.hadoop.ozone.genesis;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT;
-
import java.io.File;
import java.io.IOException;
import java.util.concurrent.locks.ReentrantLock;
-import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -38,7 +35,9 @@
import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager;
import org.apache.hadoop.hdds.scm.server.SCMConfigurator;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
-import org.apache.hadoop.security.authentication.client.AuthenticationException;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.Level;
import org.openjdk.jmh.annotations.Param;
@@ -67,7 +66,7 @@
@Setup(Level.Trial)
public static void initialize()
- throws IOException, AuthenticationException, InterruptedException {
+ throws Exception {
try {
lock.lock();
if (scm == null) {
@@ -92,7 +91,7 @@
pipelineManager.openPipeline(pipeline.getId());
}
scm.getEventQueue().fireEvent(SCMEvents.SAFE_MODE_STATUS,
- new SCMSafeModeManager.SafeModeStatus(false));
+ new SCMSafeModeManager.SafeModeStatus(false, false));
Thread.sleep(1000);
}
} finally {
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchmarkChunkManager.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchmarkChunkManager.java
index 0a80ad3..c9d2b5e 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchmarkChunkManager.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchmarkChunkManager.java
@@ -17,8 +17,14 @@
*/
package org.apache.hadoop.ozone.genesis;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
@@ -32,9 +38,15 @@
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
-import org.apache.hadoop.ozone.container.keyvalue.impl.FilePerChunkStrategy;
import org.apache.hadoop.ozone.container.keyvalue.impl.FilePerBlockStrategy;
+import org.apache.hadoop.ozone.container.keyvalue.impl.FilePerChunkStrategy;
import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import org.apache.commons.io.FileUtils;
+import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric;
+import static org.apache.hadoop.ozone.container.common.impl.ChunkLayOutVersion.FILE_PER_BLOCK;
+import static org.apache.hadoop.ozone.container.common.impl.ChunkLayOutVersion.FILE_PER_CHUNK;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.Level;
import org.openjdk.jmh.annotations.Measurement;
@@ -46,19 +58,6 @@
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.file.Files;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric;
-import static org.apache.hadoop.ozone.container.common.impl.ChunkLayOutVersion.FILE_PER_BLOCK;
-import static org.apache.hadoop.ozone.container.common.impl.ChunkLayOutVersion.FILE_PER_CHUNK;
-
/**
* Benchmark for ChunkManager implementations.
*/
@@ -99,7 +98,7 @@
private File dir;
private ChunkBuffer buffer;
private VolumeSet volumeSet;
- private Configuration config;
+ private OzoneConfiguration config;
private static File getTestDir() throws IOException {
File dir = new File(DEFAULT_TEST_DATA_DIR).getAbsoluteFile();
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java
index 25ceb4c..797c805 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java
@@ -17,29 +17,6 @@
*/
package org.apache.hadoop.ozone.genesis;
-import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.StorageUnit;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
-import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
-import org.apache.hadoop.hdds.scm.server.SCMConfigurator;
-import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
-import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
-import org.apache.hadoop.hdds.server.ServerUtils;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.common.Storage;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
-import org.apache.hadoop.ozone.om.OMStorage;
-import org.apache.hadoop.ozone.om.OzoneManager;
-import org.apache.hadoop.security.authentication.client.AuthenticationException;
-import org.apache.hadoop.hdds.utils.MetadataStore;
-import org.apache.hadoop.hdds.utils.MetadataStoreBuilder;
-
-import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -48,9 +25,31 @@
import java.util.Random;
import java.util.UUID;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_DEFAULT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_MB;
-import static org.apache.hadoop.ozone.OzoneConsts.SCM_PIPELINE_DB;
+import org.apache.hadoop.conf.StorageUnit;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.hdds.scm.server.SCMConfigurator;
+import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.hdds.utils.MetadataStore;
+import org.apache.hadoop.hdds.utils.MetadataStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.ozone.common.Storage;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.om.OMStorage;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+
+import org.apache.commons.lang3.RandomStringUtils;
+
/**
* Utility class for benchmark test cases.
@@ -77,7 +76,7 @@
public static MetadataStore getMetadataStore(String dbType)
throws IOException {
- Configuration conf = new Configuration();
+ OzoneConfiguration conf = new OzoneConfiguration();
MetadataStoreBuilder builder = MetadataStoreBuilder.newBuilder();
builder.setConf(conf);
builder.setCreateIfMissing(true);
@@ -137,7 +136,7 @@
return new StorageContainerManager(conf, configurator);
}
- static void configureSCM(Configuration conf, int numHandlers) {
+ static void configureSCM(OzoneConfiguration conf, int numHandlers) {
conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY,
RANDOM_LOCAL_ADDRESS);
conf.set(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY,
@@ -150,16 +149,11 @@
}
static void addPipelines(HddsProtos.ReplicationFactor factor,
- int numPipelines, Configuration conf) throws IOException {
- final File metaDir = ServerUtils.getScmDbDir(conf);
- final File pipelineDBPath = new File(metaDir, SCM_PIPELINE_DB);
- int cacheSize = conf.getInt(OZONE_SCM_DB_CACHE_SIZE_MB,
- OZONE_SCM_DB_CACHE_SIZE_DEFAULT);
- MetadataStore pipelineStore =
- MetadataStoreBuilder.newBuilder().setCreateIfMissing(true)
- .setConf(conf).setDbFile(pipelineDBPath)
- .setCacheSize(cacheSize * OzoneConsts.MB).build();
+ int numPipelines, ConfigurationSource conf) throws Exception {
+ DBStore dbStore = DBStoreBuilder.createDBStore(conf, new SCMDBDefinition());
+ Table<PipelineID, Pipeline> pipelineTable =
+ SCMDBDefinition.PIPELINES.getTable(dbStore);
List<DatanodeDetails> nodes = new ArrayList<>();
for (int i = 0; i < factor.getNumber(); i++) {
nodes
@@ -174,11 +168,11 @@
.setFactor(factor)
.setNodes(nodes)
.build();
- pipelineStore.put(pipeline.getId().getProtobuf().toByteArray(),
- pipeline.getProtobufMessage().toByteArray());
+ pipelineTable.put(pipeline.getId(),
+ pipeline);
}
- pipelineStore.close();
+ dbStore.close();
}
static OzoneManager getOm(OzoneConfiguration conf)
@@ -194,7 +188,7 @@
return OzoneManager.createOm(conf);
}
- static void configureOM(Configuration conf, int numHandlers) {
+ static void configureOM(OzoneConfiguration conf, int numHandlers) {
conf.set(OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY,
RANDOM_LOCAL_ADDRESS);
conf.setInt(OMConfigKeys.OZONE_OM_HANDLER_COUNT_KEY, numHandlers);
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Handler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Handler.java
new file mode 100644
index 0000000..ebf8c50
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Handler.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.shell;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.concurrent.Callable;
+
+import org.apache.hadoop.hdds.cli.GenericParentCommand;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.server.JsonUtils;
+
+import org.apache.hadoop.ozone.OzoneSecurityUtil;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.ParentCommand;
+
+/**
+ * Base class for shell commands that connect via Ozone client.
+ */
+@Command(mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class)
+@SuppressWarnings("squid:S106") // CLI
+public abstract class Handler implements Callable<Void> {
+
+ protected static final Logger LOG = LoggerFactory.getLogger(Handler.class);
+
+ private OzoneConfiguration conf;
+
+ @ParentCommand
+ private GenericParentCommand parent;
+
+ @CommandLine.Spec
+ private CommandLine.Model.CommandSpec spec;
+
+ public boolean isVerbose() {
+ return parent.isVerbose();
+ }
+
+ public OzoneConfiguration createOzoneConfiguration() {
+ return parent.createOzoneConfiguration();
+ }
+
+ protected OzoneAddress getAddress() throws OzoneClientException {
+ return new OzoneAddress();
+ }
+
+ protected abstract void execute(OzoneClient client, OzoneAddress address)
+ throws IOException, OzoneClientException;
+
+ /**
+ * Checks whether the current command should be executed or not.
+ * If it is skipped, an informational message should be output.
+ * Eg. some commands only work in secure clusters.
+ *
+ * @return true if the command should be executed
+ */
+ protected boolean isApplicable() {
+ return true;
+ }
+
+ @Override
+ public Void call() throws Exception {
+ conf = createOzoneConfiguration();
+
+ if (!isApplicable()) {
+ return null;
+ }
+
+ OzoneAddress address = getAddress();
+ try (OzoneClient client = createClient(address)) {
+ if (isVerbose()) {
+ address.print(out());
+ }
+ execute(client, address);
+ }
+
+ return null;
+ }
+
+ protected OzoneClient createClient(OzoneAddress address)
+ throws IOException, OzoneClientException {
+ return address.createClient(conf);
+ }
+
+ protected boolean securityEnabled() {
+ boolean enabled = OzoneSecurityUtil.isSecurityEnabled(conf);
+ if (!enabled) {
+ err().printf("Error: '%s' operation works only when security is " +
+ "enabled. To enable security set ozone.security.enabled to " +
+ "true.%n", spec.qualifiedName());
+ }
+ return enabled;
+ }
+
+ protected void printObjectAsJson(Object o) throws IOException {
+ out().println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(o));
+ }
+
+ protected OzoneConfiguration getConf() {
+ return conf;
+ }
+
+ protected PrintStream out() {
+ return System.out;
+ }
+
+ protected PrintStream err() {
+ return System.err;
+ }
+
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/ListOptions.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/ListOptions.java
new file mode 100644
index 0000000..4baf4f5
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/ListOptions.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell;
+
+import picocli.CommandLine;
+
+/**
+ * Common options for 'list' commands.
+ */
+public class ListOptions {
+
+ @CommandLine.Option(names = {"--length", "-l"},
+ description = "Maximum number of items to list",
+ defaultValue = "100",
+ showDefaultValue = CommandLine.Help.Visibility.ALWAYS)
+ private int limit;
+
+ @CommandLine.Option(names = {"--start", "-s"},
+ description = "The item to start the listing from.\n" +
+ "This will be excluded from the result.")
+ private String startItem;
+
+ @CommandLine.Option(names = {"--prefix", "-p"},
+ description = "Prefix to filter the items")
+ private String prefix;
+
+ public int getLimit() {
+ if (limit < 1) {
+ throw new IllegalArgumentException(
+ "List length should be a positive number");
+ }
+
+ return limit;
+ }
+
+ public String getStartItem() {
+ return startItem;
+ }
+
+ public String getPrefix() {
+ return prefix;
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/OzoneAddress.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/OzoneAddress.java
similarity index 88%
rename from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/OzoneAddress.java
rename to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/OzoneAddress.java
index 4c08ea7..1d75328 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/OzoneAddress.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/OzoneAddress.java
@@ -15,9 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.ozone.web.ozShell;
+package org.apache.hadoop.ozone.shell;
import java.io.IOException;
+import java.io.PrintStream;
import java.net.URI;
import java.net.URISyntaxException;
@@ -26,6 +27,8 @@
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientException;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
+import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_HTTP_SCHEME;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_RPC_SCHEME;
@@ -38,6 +41,7 @@
*/
public class OzoneAddress {
+ private static final int DEFAULT_OZONE_PORT = 50070;
private static final String EMPTY_HOST = "___DEFAULT___";
private URI ozoneURI;
@@ -166,8 +170,6 @@
*
* @param uri - UriString
* @return URI
- * @throws URISyntaxException
- * @throws OzoneException
*/
protected URI parseURI(String uri)
throws OzoneClientException {
@@ -177,7 +179,7 @@
}
URIBuilder uriBuilder = new URIBuilder(stringToUri(uri));
if (uriBuilder.getPort() == 0) {
- uriBuilder.setPort(Shell.DEFAULT_OZONE_PORT);
+ uriBuilder.setPort(DEFAULT_OZONE_PORT);
}
try {
@@ -298,4 +300,39 @@
"Invalid URI. Volume/bucket/key elements should not been used");
}
}
+
+ public OzoneObj toOzoneObj(OzoneObj.StoreType storeType) {
+ return OzoneObjInfo.Builder.newBuilder()
+ .setBucketName(bucketName)
+ .setVolumeName(volumeName)
+ .setKeyName(keyName)
+ .setResType(getResourceType())
+ .setStoreType(storeType)
+ .build();
+ }
+
+ private OzoneObj.ResourceType getResourceType() {
+ if (!keyName.isEmpty()) {
+ return OzoneObj.ResourceType.KEY;
+ }
+ if (!bucketName.isEmpty()) {
+ return OzoneObj.ResourceType.BUCKET;
+ }
+ if (!volumeName.isEmpty()) {
+ return OzoneObj.ResourceType.VOLUME;
+ }
+ return null;
+ }
+
+ public void print(PrintStream out) {
+ if (!volumeName.isEmpty()) {
+ out.printf("Volume Name : %s%n", volumeName);
+ }
+ if (!bucketName.isEmpty()) {
+ out.printf("Bucket Name : %s%n", bucketName);
+ }
+ if (!keyName.isEmpty()) {
+ out.printf("Key Name : %s%n", keyName);
+ }
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/OzoneShell.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/OzoneShell.java
similarity index 77%
rename from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/OzoneShell.java
rename to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/OzoneShell.java
index b97d732..7dad764 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/OzoneShell.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/OzoneShell.java
@@ -15,17 +15,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.ozone.web.ozShell;
+package org.apache.hadoop.ozone.shell;
+
+import java.util.function.Supplier;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.tracing.TracingUtil;
-import org.apache.hadoop.ozone.web.ozShell.bucket.BucketCommands;
-import org.apache.hadoop.ozone.web.ozShell.keys.KeyCommands;
-import org.apache.hadoop.ozone.web.ozShell.token.TokenCommands;
-import org.apache.hadoop.ozone.web.ozShell.volume.VolumeCommands;
+import org.apache.hadoop.ozone.shell.bucket.BucketCommands;
+import org.apache.hadoop.ozone.shell.keys.KeyCommands;
+import org.apache.hadoop.ozone.shell.token.TokenCommands;
+import org.apache.hadoop.ozone.shell.volume.VolumeCommands;
-import io.opentracing.Scope;
-import io.opentracing.util.GlobalTracer;
import picocli.CommandLine.Command;
/**
@@ -56,9 +56,11 @@
@Override
public void execute(String[] argv) {
TracingUtil.initTracing("shell", createOzoneConfiguration());
- try (Scope scope = GlobalTracer.get().buildSpan("main").startActive(true)) {
- super.execute(argv);
- }
+ TracingUtil.executeInNewSpan("main",
+ (Supplier<Void>) () -> {
+ super.execute(argv);
+ return null;
+ });
}
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Shell.java
similarity index 66%
rename from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java
rename to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Shell.java
index 999eede..0c71dc8 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Shell.java
@@ -16,14 +16,11 @@
* limitations under the License.
*/
-package org.apache.hadoop.ozone.web.ozShell;
+package org.apache.hadoop.ozone.shell;
import org.apache.hadoop.hdds.cli.GenericCli;
import org.apache.hadoop.ozone.om.exceptions.OMException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
/**
* Ozone user interface commands.
*
@@ -32,30 +29,12 @@
*/
public abstract class Shell extends GenericCli {
- private static final Logger LOG = LoggerFactory.getLogger(Shell.class);
-
public static final String OZONE_URI_DESCRIPTION = "Ozone URI could start "
- + "with o3:// or without prefix. URI may contain the host and port "
- + "of the OM server. Both are optional. "
+ + "with o3:// or without prefix. URI may contain the host/serviceId "
+ + " and port of the OM server. Both are optional. "
+ "If they are not specified it will be identified from "
+ "the config files.";
- public static final String OZONE_VOLUME_URI_DESCRIPTION =
- "URI of the volume.\n" + OZONE_URI_DESCRIPTION;
-
- public static final String OZONE_BUCKET_URI_DESCRIPTION =
- "URI of the volume/bucket.\n" + OZONE_URI_DESCRIPTION;
-
- public static final String OZONE_KEY_URI_DESCRIPTION =
- "URI of the volume/bucket/key.\n" + OZONE_URI_DESCRIPTION;
-
- public static final String OZONE_S3BUCKET_URI_DESCRIPTION = "URI of the " +
- "S3Bucket.\n" + OZONE_URI_DESCRIPTION;
-
- // General options
- public static final int DEFAULT_OZONE_PORT = 50070;
-
-
@Override
protected void printError(Throwable errorArg) {
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/StoreTypeOption.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/StoreTypeOption.java
new file mode 100644
index 0000000..8fbab64
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/StoreTypeOption.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell;
+
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
+import picocli.CommandLine;
+
+/**
+ * Option for {@link OzoneObj.StoreType}.
+ */
+public class StoreTypeOption
+ implements CommandLine.ITypeConverter<OzoneObj.StoreType> {
+
+ @CommandLine.Option(names = {"--store", "-s"},
+ description = "Store type. i.e OZONE or S3",
+ defaultValue = "OZONE",
+ converter = StoreTypeOption.class
+ )
+ private OzoneObj.StoreType value;
+
+ public OzoneObj.StoreType getValue() {
+ return value;
+ }
+
+ @Override
+ public OzoneObj.StoreType convert(String str) {
+ return str != null
+ ? OzoneObj.StoreType.valueOf(str)
+ : OzoneObj.StoreType.OZONE;
+ }
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/acl/AclHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/acl/AclHandler.java
new file mode 100644
index 0000000..50db64b
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/acl/AclHandler.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.acl;
+
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.ozone.shell.StoreTypeOption;
+import org.apache.hadoop.ozone.shell.Handler;
+
+import picocli.CommandLine;
+
+import java.io.IOException;
+
+/**
+ * Base class for ACL-related commands.
+ */
+public abstract class AclHandler extends Handler {
+
+ public static final String ADD_ACL_NAME = "addacl";
+ public static final String ADD_ACL_DESC = "Add one or more new ACLs.";
+
+ public static final String GET_ACL_NAME = "getacl";
+ public static final String GET_ACL_DESC = "List all ACLs.";
+
+ public static final String REMOVE_ACL_NAME = "removeacl";
+ public static final String REMOVE_ACL_DESC = "Remove one or more existing " +
+ "ACLs.";
+
+ public static final String SET_ACL_NAME = "setacl";
+ public static final String SET_ACL_DESC = "Set one or more ACLs, replacing " +
+ "the existing ones.";
+
+ @CommandLine.Mixin
+ private StoreTypeOption storeType;
+
+ protected abstract void execute(OzoneClient client, OzoneObj obj)
+ throws IOException;
+
+ @Override
+ protected void execute(OzoneClient client, OzoneAddress address)
+ throws IOException {
+
+ execute(client, address.toOzoneObj(storeType.getValue()));
+ }
+
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/acl/AclOption.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/acl/AclOption.java
new file mode 100644
index 0000000..aa1675d
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/acl/AclOption.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.acl;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.hadoop.ozone.OzoneAcl;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
+import picocli.CommandLine;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.List;
+
+/**
+ * Defines command-line option for specifying one or more ACLs.
+ */
+public class AclOption implements CommandLine.ITypeConverter<OzoneAcl> {
+
+ @CommandLine.Option(names = {"--acls", "--acl", "-al", "-a"}, split = ",",
+ required = true,
+ converter = AclOption.class,
+ description = "Comma separated ACL list:\n" +
+ "Example: user:user2:a OR user:user1:rw,group:hadoop:a\n" +
+ "r = READ, " +
+ "w = WRITE, " +
+ "c = CREATE, " +
+ "d = DELETE, " +
+ "l = LIST, " +
+ "a = ALL, " +
+ "n = NONE, " +
+ "x = READ_ACL, " +
+ "y = WRITE_ACL.")
+ private OzoneAcl[] values;
+
+ private List<OzoneAcl> getAclList() {
+ return ImmutableList.copyOf(values);
+ }
+
+ public void addTo(OzoneObj obj, ObjectStore objectStore, PrintStream out)
+ throws IOException {
+ for (OzoneAcl acl : getAclList()) {
+ boolean result = objectStore.addAcl(obj, acl);
+
+ String message = result
+ ? ("ACL %s added successfully.%n")
+ : ("ACL %s already exists.%n");
+
+ out.printf(message, acl);
+ }
+ }
+
+ public void removeFrom(OzoneObj obj, ObjectStore objectStore, PrintStream out)
+ throws IOException {
+ for (OzoneAcl acl : getAclList()) {
+ boolean result = objectStore.removeAcl(obj, acl);
+
+ String message = result
+ ? ("ACL %s removed successfully.%n")
+ : ("ACL %s doesn't exist.%n");
+
+ out.printf(message, acl);
+ }
+ }
+
+ public void setOn(OzoneObj obj, ObjectStore objectStore, PrintStream out)
+ throws IOException {
+ objectStore.setAcl(obj, getAclList());
+ out.println("ACLs set successfully.");
+ }
+
+ @Override
+ public OzoneAcl convert(String value) {
+ return OzoneAcl.parseAcl(value);
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/ObjectPrinter.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/acl/GetAclHandler.java
similarity index 63%
rename from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/ObjectPrinter.java
rename to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/acl/GetAclHandler.java
index 2a17275..e4ee153 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/ObjectPrinter.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/acl/GetAclHandler.java
@@ -15,24 +15,24 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.ozone.web.ozShell;
+package org.apache.hadoop.ozone.shell.acl;
+
+import org.apache.hadoop.ozone.OzoneAcl;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
import java.io.IOException;
-
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
+import java.util.List;
/**
- * Utility to print out response object in human readable form.
+ * Get ACLs.
*/
-public final class ObjectPrinter {
- private ObjectPrinter() {
+public abstract class GetAclHandler extends AclHandler {
+
+ @Override
+ protected void execute(OzoneClient client, OzoneObj obj) throws IOException {
+ List<OzoneAcl> result = client.getObjectStore().getAcl(obj);
+ printObjectAsJson(result);
}
- public static String getObjectAsJson(Object o) throws IOException {
- return JsonUtils.toJsonStringWithDefaultPrettyPrinter(o);
- }
-
- public static void printObjectAsJson(Object o) throws IOException {
- System.out.println(getObjectAsJson(o));
- }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/acl/package-info.java
similarity index 88%
copy from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/package-info.java
copy to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/acl/package-info.java
index c344c35..e569522 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/package-info.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/acl/package-info.java
@@ -18,6 +18,6 @@
*/
/**
- * Netty-based HTTP server implementation for Ozone.
+ * Helpers for ACL commands.
*/
-package org.apache.hadoop.ozone.web.ozShell.bucket;
+package org.apache.hadoop.ozone.shell.acl;
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/AddAclBucketHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/AddAclBucketHandler.java
new file mode 100644
index 0000000..90bd2f2
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/AddAclBucketHandler.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.bucket;
+
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.ozone.shell.acl.AclHandler;
+import org.apache.hadoop.ozone.shell.acl.AclOption;
+import picocli.CommandLine;
+
+import java.io.IOException;
+
+/**
+ * Add ACL to bucket.
+ */
+@CommandLine.Command(name = AclHandler.ADD_ACL_NAME,
+ description = AclHandler.ADD_ACL_DESC)
+public class AddAclBucketHandler extends AclHandler {
+
+ @CommandLine.Mixin
+ private BucketUri address;
+
+ @CommandLine.Mixin
+ private AclOption acls;
+
+ @Override
+ protected OzoneAddress getAddress() {
+ return address.getValue();
+ }
+
+ @Override
+ protected void execute(OzoneClient client, OzoneObj obj) throws IOException {
+ acls.addTo(obj, client.getObjectStore(), out());
+ }
+
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/BucketCommands.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/BucketCommands.java
similarity index 95%
rename from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/BucketCommands.java
rename to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/BucketCommands.java
index ba1ef8c..ea4ec70 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/BucketCommands.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/BucketCommands.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hadoop.ozone.web.ozShell.bucket;
+package org.apache.hadoop.ozone.shell.bucket;
import java.util.concurrent.Callable;
@@ -24,7 +24,7 @@
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.cli.MissingSubcommandException;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
+import org.apache.hadoop.ozone.shell.Shell;
import picocli.CommandLine.Command;
import picocli.CommandLine.ParentCommand;
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/GuiceInjectorUtilsForTestsImpl.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/BucketHandler.java
similarity index 66%
copy from hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/GuiceInjectorUtilsForTestsImpl.java
copy to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/BucketHandler.java
index 6f16c1c..eb496c9 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/GuiceInjectorUtilsForTestsImpl.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/BucketHandler.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -15,14 +15,23 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.hadoop.ozone.shell.bucket;
-package org.apache.hadoop.ozone.recon;
-
-import org.apache.hadoop.ozone.recon.types.GuiceInjectorUtilsForTests;
+import org.apache.hadoop.ozone.shell.Handler;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import picocli.CommandLine;
/**
- * Implementation for GuiceInjectorUtilsForTests.
+ * Base class for bucket command handlers.
*/
-public class GuiceInjectorUtilsForTestsImpl implements
- GuiceInjectorUtilsForTests {
+public abstract class BucketHandler extends Handler {
+
+ @CommandLine.Mixin
+ private BucketUri address;
+
+ @Override
+ protected OzoneAddress getAddress() {
+ return address.getValue();
+ }
+
}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/BucketUri.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/BucketUri.java
new file mode 100644
index 0000000..9a1c62d
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/BucketUri.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.bucket;
+
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.shell.Shell;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import picocli.CommandLine;
+
+/**
+ * URI parameter for bucket-specific commands.
+ */
+public class BucketUri implements CommandLine.ITypeConverter<OzoneAddress> {
+
+ private static final String OZONE_BUCKET_URI_DESCRIPTION =
+ "URI of the volume/bucket.\n" + Shell.OZONE_URI_DESCRIPTION;
+
+ @CommandLine.Parameters(index = "0", arity = "1..1",
+ description = OZONE_BUCKET_URI_DESCRIPTION,
+ converter = BucketUri.class)
+ private OzoneAddress value;
+
+ public OzoneAddress getValue() {
+ return value;
+ }
+
+ @Override
+ public OzoneAddress convert(String str) throws OzoneClientException {
+ OzoneAddress address = new OzoneAddress(str);
+ address.ensureBucketAddress();
+ return address;
+ }
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java
new file mode 100644
index 0000000..901c1e3
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.bucket;
+
+import org.apache.hadoop.hdds.protocol.StorageType;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.BucketArgs;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+
+import java.io.IOException;
+
+/**
+ * create bucket handler.
+ */
+@Command(name = "create",
+ description = "creates a bucket in a given volume")
+public class CreateBucketHandler extends BucketHandler {
+
+ @Option(names = {"--bucketkey", "-k"},
+ description = "bucket encryption key name")
+ private String bekName;
+
+ @Option(names = {"--enforcegdpr", "-g"},
+ description = "if true, indicates GDPR enforced bucket, " +
+ "false/unspecified indicates otherwise")
+ private Boolean isGdprEnforced;
+
+ /**
+ * Executes create bucket.
+ */
+ @Override
+ public void execute(OzoneClient client, OzoneAddress address)
+ throws IOException {
+
+ BucketArgs.Builder bb = new BucketArgs.Builder()
+ .setStorageType(StorageType.DEFAULT)
+ .setVersioning(false);
+
+ if (isGdprEnforced != null) {
+ bb.addMetadata(OzoneConsts.GDPR_FLAG, String.valueOf(isGdprEnforced));
+ }
+
+ if (bekName != null) {
+ if (!bekName.isEmpty()) {
+ bb.setBucketEncryptionKey(bekName);
+ } else {
+ throw new IllegalArgumentException("Bucket encryption key name must" +
+ " " + "be specified to enable bucket encryption!");
+ }
+ if (isVerbose()) {
+ out().printf("Bucket Encryption enabled with Key Name: %s%n",
+ bekName);
+ }
+ }
+
+ String volumeName = address.getVolumeName();
+ String bucketName = address.getBucketName();
+
+ OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
+ vol.createBucket(bucketName, bb.build());
+
+ if (isVerbose()) {
+ OzoneBucket bucket = vol.getBucket(bucketName);
+ printObjectAsJson(bucket);
+ }
+ }
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/DeleteBucketHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/DeleteBucketHandler.java
new file mode 100644
index 0000000..a8eeee9
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/DeleteBucketHandler.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.shell.bucket;
+
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+
+import picocli.CommandLine.Command;
+
+import java.io.IOException;
+
+/**
+ * Delete bucket Handler.
+ */
+@Command(name = "delete",
+ description = "deletes an empty bucket")
+public class DeleteBucketHandler extends BucketHandler {
+
+ @Override
+ protected void execute(OzoneClient client, OzoneAddress address)
+ throws IOException {
+
+ String volumeName = address.getVolumeName();
+ String bucketName = address.getBucketName();
+
+ OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
+ vol.deleteBucket(bucketName);
+ }
+
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/ObjectPrinter.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/GetAclBucketHandler.java
similarity index 60%
copy from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/ObjectPrinter.java
copy to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/GetAclBucketHandler.java
index 2a17275..a9a2d81 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/ObjectPrinter.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/GetAclBucketHandler.java
@@ -15,24 +15,26 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.ozone.web.ozShell;
+package org.apache.hadoop.ozone.shell.bucket;
-import java.io.IOException;
-
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.ozone.shell.acl.AclHandler;
+import org.apache.hadoop.ozone.shell.acl.GetAclHandler;
+import picocli.CommandLine;
/**
- * Utility to print out response object in human readable form.
+ * Get ACL of bucket.
*/
-public final class ObjectPrinter {
- private ObjectPrinter() {
+@CommandLine.Command(name = AclHandler.GET_ACL_NAME,
+ description = AclHandler.GET_ACL_DESC)
+public class GetAclBucketHandler extends GetAclHandler {
+
+ @CommandLine.Mixin
+ private BucketUri address;
+
+ @Override
+ protected OzoneAddress getAddress() {
+ return address.getValue();
}
- public static String getObjectAsJson(Object o) throws IOException {
- return JsonUtils.toJsonStringWithDefaultPrettyPrinter(o);
- }
-
- public static void printObjectAsJson(Object o) throws IOException {
- System.out.println(getObjectAsJson(o));
- }
}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/InfoBucketHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/InfoBucketHandler.java
new file mode 100644
index 0000000..e9ae5f9
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/InfoBucketHandler.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.bucket;
+
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+
+import picocli.CommandLine.Command;
+
+import java.io.IOException;
+
+/**
+ * Executes Info bucket.
+ */
+@Command(name = "info",
+ description = "returns information about a bucket")
+public class InfoBucketHandler extends BucketHandler {
+
+ @Override
+ public void execute(OzoneClient client, OzoneAddress address)
+ throws IOException {
+
+ OzoneBucket bucket = client.getObjectStore()
+ .getVolume(address.getVolumeName())
+ .getBucket(address.getBucketName());
+
+ printObjectAsJson(bucket);
+ }
+
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/ListBucketHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/ListBucketHandler.java
new file mode 100644
index 0000000..6c01af3
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/ListBucketHandler.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.shell.bucket;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.shell.ListOptions;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.ozone.shell.volume.VolumeHandler;
+
+import picocli.CommandLine;
+import picocli.CommandLine.Command;
+
+/**
+ * Executes List Bucket.
+ */
+@Command(name = "list",
+ aliases = "ls",
+ description = "lists the buckets in a volume.")
+public class ListBucketHandler extends VolumeHandler {
+
+ @CommandLine.Mixin
+ private ListOptions listOptions;
+
+ @Override
+ protected void execute(OzoneClient client, OzoneAddress address)
+ throws IOException {
+
+ String volumeName = address.getVolumeName();
+ OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
+ Iterator<? extends OzoneBucket> bucketIterator =
+ vol.listBuckets(listOptions.getPrefix(), listOptions.getStartItem());
+
+ int counter = 0;
+ while (listOptions.getLimit() > counter && bucketIterator.hasNext()) {
+ printObjectAsJson(bucketIterator.next());
+
+ counter++;
+ }
+
+ if (isVerbose()) {
+ out().printf("Found : %d buckets for volume : %s ",
+ counter, volumeName);
+ }
+ }
+
+}
+
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/RemoveAclBucketHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/RemoveAclBucketHandler.java
new file mode 100644
index 0000000..97a3938
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/RemoveAclBucketHandler.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.bucket;
+
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.ozone.shell.acl.AclHandler;
+import org.apache.hadoop.ozone.shell.acl.AclOption;
+import picocli.CommandLine;
+
+import java.io.IOException;
+
+/**
+ * Remove ACL from bucket.
+ */
+@CommandLine.Command(name = AclHandler.REMOVE_ACL_NAME,
+ description = AclHandler.REMOVE_ACL_DESC)
+public class RemoveAclBucketHandler extends AclHandler {
+
+ @CommandLine.Mixin
+ private BucketUri address;
+
+ @CommandLine.Mixin
+ private AclOption acls;
+
+ @Override
+ protected OzoneAddress getAddress() {
+ return address.getValue();
+ }
+
+ @Override
+ protected void execute(OzoneClient client, OzoneObj obj) throws IOException {
+ acls.removeFrom(obj, client.getObjectStore(), out());
+ }
+
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/SetAclBucketHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/SetAclBucketHandler.java
new file mode 100644
index 0000000..5b60451
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/SetAclBucketHandler.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.bucket;
+
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.ozone.shell.acl.AclHandler;
+import org.apache.hadoop.ozone.shell.acl.AclOption;
+import picocli.CommandLine;
+
+import java.io.IOException;
+
+/**
+ * Set ACL on bucket.
+ */
+@CommandLine.Command(name = AclHandler.SET_ACL_NAME,
+ description = AclHandler.SET_ACL_DESC)
+public class SetAclBucketHandler extends AclHandler {
+
+ @CommandLine.Mixin
+ private BucketUri address;
+
+ @CommandLine.Mixin
+ private AclOption acls;
+
+ @Override
+ protected OzoneAddress getAddress() {
+ return address.getValue();
+ }
+
+ @Override
+ protected void execute(OzoneClient client, OzoneObj obj)
+ throws IOException {
+ acls.setOn(obj, client.getObjectStore(), out());
+ }
+
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/package-info.java
similarity index 88%
rename from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/package-info.java
rename to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/package-info.java
index c344c35..7a971e9 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/package-info.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/package-info.java
@@ -18,6 +18,6 @@
*/
/**
- * Netty-based HTTP server implementation for Ozone.
+ * Commands for Ozone buckets.
*/
-package org.apache.hadoop.ozone.web.ozShell.bucket;
+package org.apache.hadoop.ozone.shell.bucket;
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/AddAclKeyHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/AddAclKeyHandler.java
new file mode 100644
index 0000000..ae18efd
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/AddAclKeyHandler.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.keys;
+
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.ozone.shell.acl.AclHandler;
+import org.apache.hadoop.ozone.shell.acl.AclOption;
+import picocli.CommandLine;
+
+import java.io.IOException;
+
+/**
+ * Add ACL to key.
+ */
+@CommandLine.Command(name = AclHandler.ADD_ACL_NAME,
+ description = AclHandler.ADD_ACL_DESC)
+public class AddAclKeyHandler extends AclHandler {
+
+ @CommandLine.Mixin
+ private KeyUri address;
+
+ @CommandLine.Mixin
+ private AclOption acls;
+
+ @Override
+ protected OzoneAddress getAddress() {
+ return address.getValue();
+ }
+
+ @Override
+ protected void execute(OzoneClient client, OzoneObj obj) throws IOException {
+ acls.addTo(obj, client.getObjectStore(), out());
+ }
+
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/DeleteKeyHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/DeleteKeyHandler.java
new file mode 100644
index 0000000..a62ad83
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/DeleteKeyHandler.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.shell.keys;
+
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+
+import picocli.CommandLine.Command;
+
+import java.io.IOException;
+
+/**
+ * Executes Delete Key.
+ */
+@Command(name = "delete",
+ description = "deletes an existing key")
+public class DeleteKeyHandler extends KeyHandler {
+
+ @Override
+ protected void execute(OzoneClient client, OzoneAddress address)
+ throws IOException, OzoneClientException {
+
+ String volumeName = address.getVolumeName();
+ String bucketName = address.getBucketName();
+ String keyName = address.getKeyName();
+
+ OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
+ OzoneBucket bucket = vol.getBucket(bucketName);
+ bucket.deleteKey(keyName);
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/ObjectPrinter.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/GetAclKeyHandler.java
similarity index 60%
copy from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/ObjectPrinter.java
copy to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/GetAclKeyHandler.java
index 2a17275..8b89d7a 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/ObjectPrinter.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/GetAclKeyHandler.java
@@ -15,24 +15,26 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.ozone.web.ozShell;
+package org.apache.hadoop.ozone.shell.keys;
-import java.io.IOException;
-
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.ozone.shell.acl.AclHandler;
+import org.apache.hadoop.ozone.shell.acl.GetAclHandler;
+import picocli.CommandLine;
/**
- * Utility to print out response object in human readable form.
+ * Get ACL of key.
*/
-public final class ObjectPrinter {
- private ObjectPrinter() {
+@CommandLine.Command(name = AclHandler.GET_ACL_NAME,
+ description = AclHandler.GET_ACL_DESC)
+public class GetAclKeyHandler extends GetAclHandler {
+
+ @CommandLine.Mixin
+ private KeyUri address;
+
+ @Override
+ protected OzoneAddress getAddress() {
+ return address.getValue();
}
- public static String getObjectAsJson(Object o) throws IOException {
- return JsonUtils.toJsonStringWithDefaultPrettyPrinter(o);
- }
-
- public static void printObjectAsJson(Object o) throws IOException {
- System.out.println(getObjectAsJson(o));
- }
}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/GetKeyHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/GetKeyHandler.java
new file mode 100644
index 0000000..5df236e
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/GetKeyHandler.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.shell.keys;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.hadoop.conf.StorageUnit;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY;
+
+import picocli.CommandLine;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Parameters;
+
+/**
+ * Gets an existing key.
+ */
+@Command(name = "get",
+ description = "Gets a specific key from ozone server")
+public class GetKeyHandler extends KeyHandler {
+
+ @Parameters(index = "1", arity = "1..1",
+ description = "File path to download the key to")
+ private String fileName;
+
+ @CommandLine.Option(
+ names = {"-f", "--force"},
+ description = "Overwrite local file if it exists",
+ defaultValue = "false"
+ )
+ private boolean force;
+
+ @Override
+ protected void execute(OzoneClient client, OzoneAddress address)
+ throws IOException, OzoneClientException {
+
+ String volumeName = address.getVolumeName();
+ String bucketName = address.getBucketName();
+ String keyName = address.getKeyName();
+
+ File dataFile = new File(fileName);
+
+ if (dataFile.exists() && dataFile.isDirectory()) {
+ dataFile = new File(fileName, keyName);
+ }
+
+ if (dataFile.exists() && !force) {
+ throw new OzoneClientException(dataFile.getPath() + " exists."
+ + " Download would overwrite an existing file. Aborting.");
+ }
+
+ int chunkSize = (int) getConf().getStorageSize(OZONE_SCM_CHUNK_SIZE_KEY,
+ OZONE_SCM_CHUNK_SIZE_DEFAULT, StorageUnit.BYTES);
+
+ OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
+ OzoneBucket bucket = vol.getBucket(bucketName);
+ try (InputStream input = bucket.readKey(keyName);
+ OutputStream output = new FileOutputStream(dataFile)) {
+ IOUtils.copyBytes(input, output, chunkSize);
+ }
+
+ if (isVerbose() && !"/dev/null".equals(dataFile.getAbsolutePath())) {
+ try (InputStream stream = new FileInputStream(dataFile)) {
+ String hash = DigestUtils.md5Hex(stream);
+ out().printf("Downloaded file hash : %s%n", hash);
+ }
+ }
+ }
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/InfoKeyHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/InfoKeyHandler.java
new file mode 100644
index 0000000..5cfc1e8
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/InfoKeyHandler.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.shell.keys;
+
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+
+import picocli.CommandLine.Command;
+
+import java.io.IOException;
+
+/**
+ * Executes Info Object.
+ */
+@Command(name = "info",
+ description = "returns information about an existing key")
+public class InfoKeyHandler extends KeyHandler {
+
+ @Override
+ protected void execute(OzoneClient client, OzoneAddress address)
+ throws IOException {
+
+ String volumeName = address.getVolumeName();
+ String bucketName = address.getBucketName();
+ String keyName = address.getKeyName();
+
+ OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
+ OzoneBucket bucket = vol.getBucket(bucketName);
+ OzoneKeyDetails key = bucket.getKey(keyName);
+ // For compliance/security, GDPR Secret & Algorithm details are removed
+ // from local copy of metadata before printing. This doesn't remove these
+ // from Ozone Manager's actual metadata.
+ key.getMetadata().remove(OzoneConsts.GDPR_SECRET);
+ key.getMetadata().remove(OzoneConsts.GDPR_ALGORITHM);
+
+ printObjectAsJson(key);
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/KeyCommands.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/KeyCommands.java
similarity index 95%
rename from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/KeyCommands.java
rename to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/KeyCommands.java
index 4de97c5..bc42962 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/KeyCommands.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/KeyCommands.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hadoop.ozone.web.ozShell.keys;
+package org.apache.hadoop.ozone.shell.keys;
import java.util.concurrent.Callable;
@@ -24,7 +24,7 @@
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.cli.MissingSubcommandException;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
+import org.apache.hadoop.ozone.shell.Shell;
import picocli.CommandLine.Command;
import picocli.CommandLine.ParentCommand;
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/KeyHandler.java
similarity index 66%
copy from hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/package-info.java
copy to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/KeyHandler.java
index 80c1985..861540c 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/package-info.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/KeyHandler.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -15,7 +15,23 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.ozone.web.ozShell;
+package org.apache.hadoop.ozone.shell.keys;
+
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.ozone.shell.Handler;
+import picocli.CommandLine;
+
/**
- * Tests for ozone shell..
+ * Base class for key command handlers.
*/
+public abstract class KeyHandler extends Handler {
+
+ @CommandLine.Mixin
+ private KeyUri address;
+
+ @Override
+ protected OzoneAddress getAddress() {
+ return address.getValue();
+ }
+
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/KeyUri.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/KeyUri.java
new file mode 100644
index 0000000..b2caf0d
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/KeyUri.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.keys;
+
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.shell.Shell;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import picocli.CommandLine;
+
+/**
+ * URI parameter for key-specific commands.
+ */
+public class KeyUri implements CommandLine.ITypeConverter<OzoneAddress> {
+
+ private static final String OZONE_KEY_URI_DESCRIPTION =
+ "URI of the volume/bucket/key.\n" + Shell.OZONE_URI_DESCRIPTION;
+
+ @CommandLine.Parameters(index = "0", arity = "1..1",
+ description = OZONE_KEY_URI_DESCRIPTION,
+ converter = KeyUri.class)
+ private OzoneAddress value;
+
+ public OzoneAddress getValue() {
+ return value;
+ }
+
+ @Override
+ public OzoneAddress convert(String str) throws OzoneClientException {
+ OzoneAddress address = new OzoneAddress(str);
+ address.ensureKeyAddress();
+ return address;
+ }
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ListKeyHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ListKeyHandler.java
new file mode 100644
index 0000000..00d04fb
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ListKeyHandler.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.shell.keys;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.shell.ListOptions;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.ozone.shell.bucket.BucketHandler;
+
+import picocli.CommandLine;
+import picocli.CommandLine.Command;
+
+/**
+ * Executes List Keys.
+ */
+@Command(name = "list",
+ aliases = "ls",
+ description = "list all keys in a given bucket")
+public class ListKeyHandler extends BucketHandler {
+
+ @CommandLine.Mixin
+ private ListOptions listOptions;
+
+ @Override
+ protected void execute(OzoneClient client, OzoneAddress address)
+ throws IOException, OzoneClientException {
+
+ String volumeName = address.getVolumeName();
+ String bucketName = address.getBucketName();
+
+ OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
+ OzoneBucket bucket = vol.getBucket(bucketName);
+ Iterator<? extends OzoneKey> keyIterator = bucket.listKeys(
+ listOptions.getPrefix(), listOptions.getStartItem());
+
+ int maxKeyLimit = listOptions.getLimit();
+
+ int counter = 0;
+ while (maxKeyLimit > counter && keyIterator.hasNext()) {
+ OzoneKey ozoneKey = keyIterator.next();
+ printObjectAsJson(ozoneKey);
+ counter++;
+ }
+
+ // More keys were returned notify about max length
+ if (keyIterator.hasNext()) {
+ out().println("Listing first " + maxKeyLimit + " entries of the " +
+ "result. Use --length (-l) to override max returned keys.");
+ } else if (isVerbose()) {
+ out().printf("Found : %d keys for bucket %s in volume : %s ",
+ counter, bucketName, volumeName);
+ }
+ }
+
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java
new file mode 100644
index 0000000..2b2de5d
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.shell.keys;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.conf.StorageUnit;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+import picocli.CommandLine.Parameters;
+
+/**
+ * Puts a file into an ozone bucket.
+ */
+@Command(name = "put",
+ description = "creates or overwrites an existing key")
+public class PutKeyHandler extends KeyHandler {
+
+ @Parameters(index = "1", arity = "1..1", description = "File to upload")
+ private String fileName;
+
+ @Option(names = {"-r", "--replication"},
+ description = "Replication factor of the new key. (use ONE or THREE) "
+ + "Default is specified in the cluster-wide config.")
+ private ReplicationFactor replicationFactor;
+
+ @Override
+ protected void execute(OzoneClient client, OzoneAddress address)
+ throws IOException, OzoneClientException {
+
+ String volumeName = address.getVolumeName();
+ String bucketName = address.getBucketName();
+ String keyName = address.getKeyName();
+
+ File dataFile = new File(fileName);
+
+ if (isVerbose()) {
+ try (InputStream stream = new FileInputStream(dataFile)) {
+ String hash = DigestUtils.md5Hex(stream);
+ out().printf("File Hash : %s%n", hash);
+ }
+ }
+
+ if (replicationFactor == null) {
+ replicationFactor = ReplicationFactor.valueOf(
+ getConf().getInt(OZONE_REPLICATION, OZONE_REPLICATION_DEFAULT));
+ }
+
+ ReplicationType replicationType = ReplicationType.valueOf(
+ getConf().get(OZONE_REPLICATION_TYPE, OZONE_REPLICATION_TYPE_DEFAULT));
+ OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
+ OzoneBucket bucket = vol.getBucket(bucketName);
+
+ Map<String, String> keyMetadata = new HashMap<>();
+ String gdprEnabled = bucket.getMetadata().get(OzoneConsts.GDPR_FLAG);
+ if (Boolean.parseBoolean(gdprEnabled)) {
+ keyMetadata.put(OzoneConsts.GDPR_FLAG, Boolean.TRUE.toString());
+ }
+
+ int chunkSize = (int) getConf().getStorageSize(OZONE_SCM_CHUNK_SIZE_KEY,
+ OZONE_SCM_CHUNK_SIZE_DEFAULT, StorageUnit.BYTES);
+ try (InputStream input = new FileInputStream(dataFile);
+ OutputStream output = bucket.createKey(keyName, dataFile.length(),
+ replicationType, replicationFactor, keyMetadata)) {
+ IOUtils.copyBytes(input, output, chunkSize);
+ }
+ }
+
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/RemoveAclKeyHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/RemoveAclKeyHandler.java
new file mode 100644
index 0000000..57cf122
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/RemoveAclKeyHandler.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.keys;
+
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.ozone.shell.acl.AclHandler;
+import org.apache.hadoop.ozone.shell.acl.AclOption;
+import picocli.CommandLine;
+
+import java.io.IOException;
+
+/**
+ * Remove ACL from keys.
+ */
+@CommandLine.Command(name = AclHandler.REMOVE_ACL_NAME,
+ description = AclHandler.REMOVE_ACL_DESC)
+public class RemoveAclKeyHandler extends AclHandler {
+
+ @CommandLine.Mixin
+ private KeyUri address;
+
+ @CommandLine.Mixin
+ private AclOption acls;
+
+ @Override
+ protected OzoneAddress getAddress() {
+ return address.getValue();
+ }
+
+ @Override
+ protected void execute(OzoneClient client, OzoneObj obj) throws IOException {
+ acls.removeFrom(obj, client.getObjectStore(), out());
+ }
+
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/RenameKeyHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/RenameKeyHandler.java
new file mode 100644
index 0000000..f71ac09
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/RenameKeyHandler.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.keys;
+
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.ozone.shell.bucket.BucketHandler;
+
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Parameters;
+
+import java.io.IOException;
+
+/**
+ * Renames an existing key.
+ */
+@Command(name = "rename",
+ description = "renames an existing key")
+public class RenameKeyHandler extends BucketHandler {
+
+ @Parameters(index = "1", arity = "1..1",
+ description = "The existing key to be renamed")
+ private String fromKey;
+
+ @Parameters(index = "2", arity = "1..1",
+ description = "The new desired name of the key")
+ private String toKey;
+
+ @Override
+ protected void execute(OzoneClient client, OzoneAddress address)
+ throws IOException, OzoneClientException {
+
+ String volumeName = address.getVolumeName();
+ String bucketName = address.getBucketName();
+
+ OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
+ OzoneBucket bucket = vol.getBucket(bucketName);
+ bucket.renameKey(fromKey, toKey);
+
+ if (isVerbose()) {
+ out().printf("Renamed Key : %s to %s%n", fromKey, toKey);
+ }
+ }
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/SetAclKeyHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/SetAclKeyHandler.java
new file mode 100644
index 0000000..8d042d3
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/SetAclKeyHandler.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.keys;
+
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.ozone.shell.acl.AclHandler;
+import org.apache.hadoop.ozone.shell.acl.AclOption;
+import picocli.CommandLine;
+
+import java.io.IOException;
+
+/**
+ * Set ACL on keys.
+ */
+@CommandLine.Command(name = AclHandler.SET_ACL_NAME,
+ description = AclHandler.SET_ACL_DESC)
+public class SetAclKeyHandler extends AclHandler {
+
+ @CommandLine.Mixin
+ private KeyUri address;
+
+ @CommandLine.Mixin
+ private AclOption acls;
+
+ @Override
+ protected OzoneAddress getAddress() {
+ return address.getValue();
+ }
+
+ @Override
+ protected void execute(OzoneClient client, OzoneObj obj)
+ throws IOException {
+ acls.setOn(obj, client.getObjectStore(), out());
+ }
+
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/package-info.java
similarity index 88%
copy from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/package-info.java
copy to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/package-info.java
index c344c35..a7626ba 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/package-info.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/package-info.java
@@ -18,6 +18,6 @@
*/
/**
- * Netty-based HTTP server implementation for Ozone.
+ * Commands for Ozone keys.
*/
-package org.apache.hadoop.ozone.web.ozShell.bucket;
+package org.apache.hadoop.ozone.shell.keys;
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/s3/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/package-info.java
similarity index 90%
copy from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/s3/package-info.java
copy to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/package-info.java
index 079ef18..dc2d7ab 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/s3/package-info.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/package-info.java
@@ -15,7 +15,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
/**
- * S3 commands for Ozone.
- */
-package org.apache.hadoop.ozone.web.ozShell.s3;
+ A simple CLI to work against Ozone.
+ **/
+package org.apache.hadoop.ozone.shell;
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/s3/GetS3SecretHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/s3/GetS3SecretHandler.java
new file mode 100644
index 0000000..f0b9f2d
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/s3/GetS3SecretHandler.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.s3;
+
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.security.UserGroupInformation;
+import picocli.CommandLine.Command;
+
+import java.io.IOException;
+
+/**
+ * Executes getsecret calls.
+ */
+@Command(name = "getsecret",
+ description = "Returns s3 secret for current user")
+public class GetS3SecretHandler extends S3Handler {
+
+ @Override
+ protected boolean isApplicable() {
+ return securityEnabled();
+ }
+
+ @Override
+ protected void execute(OzoneClient client, OzoneAddress address)
+ throws IOException {
+ String userName = UserGroupInformation.getCurrentUser().getUserName();
+ out().println(client.getObjectStore().getS3Secret(userName));
+ }
+
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/s3/S3Handler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/s3/S3Handler.java
similarity index 64%
rename from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/s3/S3Handler.java
rename to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/s3/S3Handler.java
index 46e25c3..34e3b9a 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/s3/S3Handler.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/s3/S3Handler.java
@@ -16,21 +16,21 @@
* limitations under the License.
*/
-package org.apache.hadoop.ozone.web.ozShell.s3;
+package org.apache.hadoop.ozone.shell.s3;
-import org.apache.hadoop.hdds.cli.HddsVersionProvider;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import java.io.IOException;
+
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.ozone.shell.Handler;
+
import picocli.CommandLine;
/**
* Common interface for S3 command handling.
*/
-@CommandLine.Command(mixinStandardHelpOptions = true,
- versionProvider = HddsVersionProvider.class)
-public class S3Handler extends Handler {
- protected static final Logger LOG = LoggerFactory.getLogger(S3Handler.class);
+public abstract class S3Handler extends Handler {
@CommandLine.Option(names = {"--om-service-id"},
required = false,
@@ -41,4 +41,16 @@
public String getOmServiceID() {
return omServiceID;
}
+
+ @Override
+ protected OzoneAddress getAddress() throws OzoneClientException {
+ return new OzoneAddress();
+ }
+
+ @Override
+ protected OzoneClient createClient(OzoneAddress address)
+ throws IOException, OzoneClientException {
+ return address.createClientForS3Commands(getConf(), omServiceID);
+ }
+
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/s3/S3Shell.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/s3/S3Shell.java
similarity index 79%
rename from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/s3/S3Shell.java
rename to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/s3/S3Shell.java
index 73f5d53..4116372 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/s3/S3Shell.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/s3/S3Shell.java
@@ -15,12 +15,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.ozone.web.ozShell.s3;
+package org.apache.hadoop.ozone.shell.s3;
-import io.opentracing.Scope;
-import io.opentracing.util.GlobalTracer;
+import java.util.function.Supplier;
+
import org.apache.hadoop.hdds.tracing.TracingUtil;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
+import org.apache.hadoop.ozone.shell.Shell;
+
import picocli.CommandLine.Command;
/**
@@ -29,18 +30,18 @@
@Command(name = "ozone s3",
description = "Shell for S3 specific operations",
subcommands = {
- GetS3SecretHandler.class,
- S3BucketMapping.class
+ GetS3SecretHandler.class
})
-
public class S3Shell extends Shell {
@Override
public void execute(String[] argv) {
TracingUtil.initTracing("s3shell", createOzoneConfiguration());
- try (Scope scope = GlobalTracer.get().buildSpan("main").startActive(true)) {
- super.execute(argv);
- }
+ TracingUtil.executeInNewSpan("s3shell",
+ (Supplier<Void>) () -> {
+ super.execute(argv);
+ return null;
+ });
}
/**
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/s3/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/s3/package-info.java
similarity index 94%
rename from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/s3/package-info.java
rename to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/s3/package-info.java
index 079ef18..9a5eac7 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/s3/package-info.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/s3/package-info.java
@@ -18,4 +18,4 @@
/**
* S3 commands for Ozone.
*/
-package org.apache.hadoop.ozone.web.ozShell.s3;
+package org.apache.hadoop.ozone.shell.s3;
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/CancelTokenHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/CancelTokenHandler.java
new file mode 100644
index 0000000..73c9264
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/CancelTokenHandler.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.shell.token;
+
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import picocli.CommandLine.Command;
+
+import java.io.IOException;
+
+/**
+ * Executes cancelDelegationToken api.
+ */
+@Command(name = "cancel",
+ description = "cancel a delegation token.")
+public class CancelTokenHandler extends TokenHandler {
+
+ @Override
+ protected void execute(OzoneClient client, OzoneAddress address)
+ throws IOException, OzoneClientException {
+ client.getObjectStore().cancelDelegationToken(getToken());
+ out().printf("Token canceled successfully.%n");
+ }
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/GetTokenHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/GetTokenHandler.java
new file mode 100644
index 0000000..5bf8ccb
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/GetTokenHandler.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.shell.token;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.ozone.shell.Handler;
+import org.apache.hadoop.ozone.shell.Shell;
+import org.apache.hadoop.security.token.Token;
+import picocli.CommandLine;
+import picocli.CommandLine.Command;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Executes getDelegationToken api.
+ */
+@Command(name = "get",
+ description = "get a delegation token.")
+public class GetTokenHandler extends Handler {
+
+ @CommandLine.Parameters(arity = "0..1",
+ description = Shell.OZONE_URI_DESCRIPTION)
+ private String uri;
+
+ @CommandLine.Mixin
+ private RenewerOption renewer;
+
+ @Override
+ protected OzoneAddress getAddress() throws OzoneClientException {
+ return new OzoneAddress(uri);
+ }
+
+ @Override
+ protected boolean isApplicable() {
+ return securityEnabled();
+ }
+
+ @Override
+ protected void execute(OzoneClient client, OzoneAddress address)
+ throws IOException, OzoneClientException {
+
+ Token<OzoneTokenIdentifier> token = client.getObjectStore()
+ .getDelegationToken(new Text(renewer.getValue()));
+ if (Objects.isNull(token)) {
+ err().println("Error: Get delegation token operation failed. " +
+ "Check OzoneManager logs for more details.");
+ } else {
+ printObjectAsJson(token.encodeToUrlString());
+ }
+ }
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/PrintTokenHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/PrintTokenHandler.java
new file mode 100644
index 0000000..632a9d2
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/PrintTokenHandler.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.shell.token;
+
+import org.apache.hadoop.hdds.server.JsonUtils;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+
+import picocli.CommandLine;
+import picocli.CommandLine.Command;
+
+import java.util.concurrent.Callable;
+
+/**
+ * Prints token decoded from file.
+ */
+@Command(name = "print",
+ description = "print a delegation token.")
+@SuppressWarnings("squid:S106") // CLI
+public class PrintTokenHandler implements Callable<Void> {
+
+ @CommandLine.Mixin
+ private TokenOption tokenFile;
+
+ @Override
+ public Void call() throws Exception {
+ if (tokenFile.exists()) {
+ Token<OzoneTokenIdentifier> token = tokenFile.decode();
+ System.out.print(JsonUtils.toJsonStringWithDefaultPrettyPrinter(token));
+ }
+ return null;
+ }
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/RenewTokenHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/RenewTokenHandler.java
new file mode 100644
index 0000000..6b5e7a1
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/RenewTokenHandler.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.shell.token;
+
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import picocli.CommandLine.Command;
+
+import java.io.IOException;
+
+/**
+ * Executes renewDelegationToken api.
+ */
+@Command(name = "renew",
+ description = "renew a delegation token.")
+public class RenewTokenHandler extends TokenHandler {
+
+ @Override
+ protected void execute(OzoneClient client, OzoneAddress address)
+ throws IOException, OzoneClientException {
+ long expiryTime = client.getObjectStore().renewDelegationToken(getToken());
+ out().printf("Token renewed successfully, expiry time: %s.%n", expiryTime);
+ }
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/RenewerOption.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/RenewerOption.java
new file mode 100644
index 0000000..67b8e33
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/RenewerOption.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.token;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import picocli.CommandLine;
+
+import java.io.IOException;
+
+/**
+ * Option for token renewer.
+ */
+public class RenewerOption {
+
+ @CommandLine.Option(names = {"--renewer", "-r"},
+ description = "Token renewer",
+ showDefaultValue = CommandLine.Help.Visibility.ALWAYS)
+ private String renewer;
+
+ public String getValue() throws IOException {
+ if (StringUtils.isEmpty(renewer)) {
+ renewer = UserGroupInformation.getCurrentUser().getShortUserName();
+ }
+ return renewer;
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/token/TokenCommands.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/TokenCommands.java
similarity index 94%
rename from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/token/TokenCommands.java
rename to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/TokenCommands.java
index 2501ad9..e0836ca 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/token/TokenCommands.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/TokenCommands.java
@@ -16,13 +16,13 @@
* limitations under the License.
*/
-package org.apache.hadoop.ozone.web.ozShell.token;
+package org.apache.hadoop.ozone.shell.token;
import org.apache.hadoop.hdds.cli.GenericParentCommand;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.cli.MissingSubcommandException;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
+import org.apache.hadoop.ozone.shell.Shell;
import picocli.CommandLine.Command;
import picocli.CommandLine.ParentCommand;
@@ -61,4 +61,5 @@
public OzoneConfiguration createOzoneConfiguration() {
return shell.createOzoneConfiguration();
}
+
}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/TokenHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/TokenHandler.java
new file mode 100644
index 0000000..6d8ac39
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/TokenHandler.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.token;
+
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.ozone.shell.Handler;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.security.token.Token;
+import picocli.CommandLine;
+
+import java.io.IOException;
+
+/**
+ * Handler for requests with an existing token.
+ */
+public abstract class TokenHandler extends Handler {
+
+ @CommandLine.Mixin
+ private TokenOption tokenFile;
+ private Token<OzoneTokenIdentifier> token;
+
+ @Override
+ protected boolean isApplicable() {
+ return securityEnabled() && tokenFile.exists();
+ }
+
+ @Override
+ protected OzoneClient createClient(OzoneAddress address)
+ throws IOException {
+ token = tokenFile.decode();
+ return OzoneClientFactory.getOzoneClient(getConf(), token);
+ }
+
+ Token<OzoneTokenIdentifier> getToken() {
+ return token;
+ }
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/TokenOption.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/TokenOption.java
new file mode 100644
index 0000000..61d479b
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/TokenOption.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.token;
+
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import picocli.CommandLine;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+
+/**
+ * Option for token file.
+ */
+public class TokenOption {
+
+ @CommandLine.Option(names = {"--token", "-t"},
+ description = "file containing encoded token",
+ defaultValue = "/tmp/token.txt",
+ showDefaultValue = CommandLine.Help.Visibility.ALWAYS)
+ private File tokenFile;
+
+ public boolean exists() {
+ boolean exists = tokenFile != null && tokenFile.exists();
+ if (!exists) {
+ System.err.println("Error: token operation failed as token file: "
+ + tokenFile + " containing encoded token doesn't exist.");
+ }
+ return exists;
+ }
+
+ public Token<OzoneTokenIdentifier> decode() throws IOException {
+ Token<OzoneTokenIdentifier> token = new Token<>();
+ token.decodeFromUrlString(new String(Files.readAllBytes(tokenFile.toPath()),
+ StandardCharsets.UTF_8));
+ return token;
+ }
+
+}
\ No newline at end of file
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/s3/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/package-info.java
similarity index 90%
copy from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/s3/package-info.java
copy to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/package-info.java
index 079ef18..a05c27a 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/s3/package-info.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/token/package-info.java
@@ -15,7 +15,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
/**
- * S3 commands for Ozone.
- */
-package org.apache.hadoop.ozone.web.ozShell.s3;
+ Ozone delegation token commands.
+ **/
+package org.apache.hadoop.ozone.shell.token;
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/AddAclVolumeHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/AddAclVolumeHandler.java
new file mode 100644
index 0000000..94f44b2
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/AddAclVolumeHandler.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.volume;
+
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.ozone.shell.acl.AclHandler;
+import org.apache.hadoop.ozone.shell.acl.AclOption;
+import picocli.CommandLine;
+
+import java.io.IOException;
+
+/**
+ * Add ACL to volume.
+ */
+@CommandLine.Command(name = AclHandler.ADD_ACL_NAME,
+ description = AclHandler.ADD_ACL_DESC)
+public class AddAclVolumeHandler extends AclHandler {
+
+ @CommandLine.Mixin
+ private VolumeUri address;
+
+ @CommandLine.Mixin
+ private AclOption acls;
+
+ @Override
+ protected OzoneAddress getAddress() {
+ return address.getValue();
+ }
+
+ @Override
+ protected void execute(OzoneClient client, OzoneObj obj) throws IOException {
+ acls.addTo(obj, client.getObjectStore(), out());
+ }
+
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/CreateVolumeHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/CreateVolumeHandler.java
new file mode 100644
index 0000000..cbeb92a
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/CreateVolumeHandler.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.shell.volume;
+
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.VolumeArgs;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.security.UserGroupInformation;
+
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+
+import java.io.IOException;
+
+/**
+ * Executes the create volume call for the shell.
+ */
+@Command(name = "create",
+ description = "Creates a volume for the specified user")
+public class CreateVolumeHandler extends VolumeHandler {
+
+ @Option(names = {"--user", "-u"},
+ description = "Owner of of the volume")
+ private String ownerName;
+
+ @Option(names = {"--quota", "-q"},
+ description =
+ "Quota of the newly created volume (eg. 1G)")
+ private String quota;
+
+ @Override
+ protected void execute(OzoneClient client, OzoneAddress address)
+ throws IOException {
+ if (ownerName == null) {
+ ownerName = UserGroupInformation.getCurrentUser().getUserName();
+ }
+
+ String volumeName = address.getVolumeName();
+
+ String adminName = UserGroupInformation.getCurrentUser().getUserName();
+ VolumeArgs.Builder volumeArgsBuilder = VolumeArgs.newBuilder()
+ .setAdmin(adminName)
+ .setOwner(ownerName);
+ if (quota != null) {
+ volumeArgsBuilder.setQuota(quota);
+ }
+ client.getObjectStore().createVolume(volumeName,
+ volumeArgsBuilder.build());
+
+ if (isVerbose()) {
+ OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
+ printObjectAsJson(vol);
+ }
+ }
+
+}
+
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java
new file mode 100644
index 0000000..29f9e39
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.shell.volume;
+
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+
+import picocli.CommandLine.Command;
+
+import java.io.IOException;
+
+/**
+ * Executes deleteVolume call for the shell.
+ */
+@Command(name = "delete",
+ description = "deletes a volume if it is empty")
+public class DeleteVolumeHandler extends VolumeHandler {
+
+ @Override
+ protected void execute(OzoneClient client, OzoneAddress address)
+ throws IOException {
+
+ String volumeName = address.getVolumeName();
+
+ client.getObjectStore().deleteVolume(volumeName);
+ out().printf("Volume %s is deleted%n", volumeName);
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/ObjectPrinter.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/GetAclVolumeHandler.java
similarity index 60%
copy from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/ObjectPrinter.java
copy to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/GetAclVolumeHandler.java
index 2a17275..2deb25a 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/ObjectPrinter.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/GetAclVolumeHandler.java
@@ -15,24 +15,26 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.ozone.web.ozShell;
+package org.apache.hadoop.ozone.shell.volume;
-import java.io.IOException;
-
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.ozone.shell.acl.AclHandler;
+import org.apache.hadoop.ozone.shell.acl.GetAclHandler;
+import picocli.CommandLine;
/**
- * Utility to print out response object in human readable form.
+ * Get ACL of volume.
*/
-public final class ObjectPrinter {
- private ObjectPrinter() {
+@CommandLine.Command(name = AclHandler.GET_ACL_NAME,
+ description = AclHandler.GET_ACL_DESC)
+public class GetAclVolumeHandler extends GetAclHandler {
+
+ @CommandLine.Mixin
+ private VolumeUri address;
+
+ @Override
+ protected OzoneAddress getAddress() {
+ return address.getValue();
}
- public static String getObjectAsJson(Object o) throws IOException {
- return JsonUtils.toJsonStringWithDefaultPrettyPrinter(o);
- }
-
- public static void printObjectAsJson(Object o) throws IOException {
- System.out.println(getObjectAsJson(o));
- }
}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/InfoVolumeHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/InfoVolumeHandler.java
new file mode 100644
index 0000000..db689fa
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/InfoVolumeHandler.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.shell.volume;
+
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+
+import picocli.CommandLine.Command;
+
+import java.io.IOException;
+
+/**
+ * Executes volume Info calls.
+ */
+@Command(name = "info",
+ description = "returns information about a specific volume")
+public class InfoVolumeHandler extends VolumeHandler {
+
+ @Override
+ protected void execute(OzoneClient client, OzoneAddress address)
+ throws IOException {
+
+ OzoneVolume vol = client.getObjectStore()
+ .getVolume(address.getVolumeName());
+ printObjectAsJson(vol);
+ }
+
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/ListVolumeHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/ListVolumeHandler.java
new file mode 100644
index 0000000..fb0760e
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/ListVolumeHandler.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.shell.volume;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.shell.ListOptions;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.ozone.shell.Shell;
+import org.apache.hadoop.ozone.shell.Handler;
+import org.apache.hadoop.security.UserGroupInformation;
+
+import picocli.CommandLine;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+import picocli.CommandLine.Parameters;
+
+/**
+ * Executes List Volume call.
+ */
+@Command(name = "list",
+ aliases = "ls",
+ description = "List the volumes of a given user")
+public class ListVolumeHandler extends Handler {
+
+ @Parameters(arity = "1..1",
+ description = Shell.OZONE_URI_DESCRIPTION,
+ defaultValue = "/")
+ private String uri;
+
+ @CommandLine.Mixin
+ private ListOptions listOptions;
+
+ @Option(names = {"--user", "-u"},
+ description = "List accessible volumes of the user. This will be ignored"
+ + " if list all volumes option is specified.")
+ private String userName;
+
+ @Option(names = {"--all", "-a"},
+ description = "List all volumes.")
+ private boolean listAllVolumes;
+
+ @Override
+ protected OzoneAddress getAddress() throws OzoneClientException {
+ OzoneAddress address = new OzoneAddress(uri);
+ address.ensureRootAddress();
+ return address;
+ }
+
+ @Override
+ protected void execute(OzoneClient client, OzoneAddress address)
+ throws IOException {
+
+ if (userName == null) {
+ userName = UserGroupInformation.getCurrentUser().getUserName();
+ }
+
+ Iterator<? extends OzoneVolume> volumeIterator;
+ if (userName != null && !listAllVolumes) {
+ volumeIterator = client.getObjectStore().listVolumesByUser(userName,
+ listOptions.getPrefix(), listOptions.getStartItem());
+ } else {
+ volumeIterator = client.getObjectStore().listVolumes(
+ listOptions.getPrefix(), listOptions.getStartItem());
+ }
+
+ int counter = 0;
+ while (listOptions.getLimit() > counter && volumeIterator.hasNext()) {
+ printObjectAsJson(volumeIterator.next());
+ counter++;
+ }
+
+ if (isVerbose()) {
+ out().printf("Found : %d volumes for user : %s ", counter,
+ userName);
+ }
+ }
+}
+
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/RemoveAclVolumeHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/RemoveAclVolumeHandler.java
new file mode 100644
index 0000000..32b6c43
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/RemoveAclVolumeHandler.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.volume;
+
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.ozone.shell.acl.AclHandler;
+import org.apache.hadoop.ozone.shell.acl.AclOption;
+import picocli.CommandLine;
+
+import java.io.IOException;
+
+/**
+ * Remove ACL from volume.
+ */
+@CommandLine.Command(name = AclHandler.REMOVE_ACL_NAME,
+ description = AclHandler.REMOVE_ACL_DESC)
+public class RemoveAclVolumeHandler extends AclHandler {
+
+ @CommandLine.Mixin
+ private VolumeUri address;
+
+ @CommandLine.Mixin
+ private AclOption acls;
+
+ @Override
+ protected OzoneAddress getAddress() {
+ return address.getValue();
+ }
+
+ @Override
+ protected void execute(OzoneClient client, OzoneObj obj) throws IOException {
+ acls.removeFrom(obj, client.getObjectStore(), out());
+ }
+
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/SetAclVolumeHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/SetAclVolumeHandler.java
new file mode 100644
index 0000000..54555ac
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/SetAclVolumeHandler.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.volume;
+
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.ozone.shell.acl.AclHandler;
+import org.apache.hadoop.ozone.shell.acl.AclOption;
+import picocli.CommandLine;
+
+import java.io.IOException;
+
+/**
+ * Set ACL on volume.
+ */
+@CommandLine.Command(name = AclHandler.SET_ACL_NAME,
+ description = AclHandler.SET_ACL_DESC)
+public class SetAclVolumeHandler extends AclHandler {
+
+ @CommandLine.Mixin
+ private VolumeUri address;
+
+ @CommandLine.Mixin
+ private AclOption acls;
+
+ @Override
+ protected OzoneAddress getAddress() {
+ return address.getValue();
+ }
+
+ @Override
+ protected void execute(OzoneClient client, OzoneObj obj)
+ throws IOException {
+ acls.setOn(obj, client.getObjectStore(), out());
+ }
+
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/UpdateVolumeHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/UpdateVolumeHandler.java
new file mode 100644
index 0000000..52ba5f0
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/UpdateVolumeHandler.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.shell.volume;
+
+import org.apache.hadoop.hdds.client.OzoneQuota;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+
+import java.io.IOException;
+
+/**
+ * Executes update volume calls.
+ */
+@Command(name = "update",
+ description = "Updates parameter of the volumes")
+public class UpdateVolumeHandler extends VolumeHandler {
+
+ @Option(names = {"--user"},
+ description = "Owner of the volume to set")
+ private String ownerName;
+
+ @Option(names = {"--quota"},
+ description = "Quota of the volume to set"
+ + "(eg. 1G)")
+ private String quota;
+
+ @Override
+ protected void execute(OzoneClient client, OzoneAddress address)
+ throws IOException {
+
+ String volumeName = address.getVolumeName();
+
+ OzoneVolume volume = client.getObjectStore().getVolume(volumeName);
+ if (quota != null && !quota.isEmpty()) {
+ volume.setQuota(OzoneQuota.parseQuota(quota));
+ }
+
+ if (ownerName != null && !ownerName.isEmpty()) {
+ boolean result = volume.setOwner(ownerName);
+ if (LOG.isDebugEnabled() && !result) {
+ out().format("Volume '%s' owner is already '%s'. Unchanged.%n",
+ volumeName, ownerName);
+ }
+ }
+
+ printObjectAsJson(volume);
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/VolumeCommands.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/VolumeCommands.java
similarity index 95%
rename from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/VolumeCommands.java
rename to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/VolumeCommands.java
index 833457b..1b05b48 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/VolumeCommands.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/VolumeCommands.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hadoop.ozone.web.ozShell.volume;
+package org.apache.hadoop.ozone.shell.volume;
import java.util.concurrent.Callable;
@@ -24,7 +24,7 @@
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.cli.MissingSubcommandException;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
+import org.apache.hadoop.ozone.shell.Shell;
import picocli.CommandLine.Command;
import picocli.CommandLine.ParentCommand;
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/GuiceInjectorUtilsForTestsImpl.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/VolumeHandler.java
similarity index 66%
rename from hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/GuiceInjectorUtilsForTestsImpl.java
rename to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/VolumeHandler.java
index 6f16c1c..b435e75 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/GuiceInjectorUtilsForTestsImpl.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/VolumeHandler.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -15,14 +15,23 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.hadoop.ozone.shell.volume;
-package org.apache.hadoop.ozone.recon;
-
-import org.apache.hadoop.ozone.recon.types.GuiceInjectorUtilsForTests;
+import org.apache.hadoop.ozone.shell.Handler;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import picocli.CommandLine;
/**
- * Implementation for GuiceInjectorUtilsForTests.
+ * Base class for volume command handlers.
*/
-public class GuiceInjectorUtilsForTestsImpl implements
- GuiceInjectorUtilsForTests {
+public abstract class VolumeHandler extends Handler {
+
+ @CommandLine.Mixin
+ private VolumeUri address;
+
+ @Override
+ protected OzoneAddress getAddress() {
+ return address.getValue();
+ }
+
}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/VolumeUri.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/VolumeUri.java
new file mode 100644
index 0000000..3258125
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/VolumeUri.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell.volume;
+
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.ozone.shell.Shell;
+import picocli.CommandLine;
+
+/**
+ * URI parameter for volume-specific commands.
+ */
+public class VolumeUri implements CommandLine.ITypeConverter<OzoneAddress> {
+
+ private static final String OZONE_VOLUME_URI_DESCRIPTION =
+ "URI of the volume.\n" + Shell.OZONE_URI_DESCRIPTION;
+
+ @CommandLine.Parameters(index = "0", arity = "1..1",
+ description = OZONE_VOLUME_URI_DESCRIPTION,
+ converter = VolumeUri.class)
+ private OzoneAddress value;
+
+ public OzoneAddress getValue() {
+ return value;
+ }
+
+ @Override
+ public OzoneAddress convert(String str) throws OzoneClientException {
+ OzoneAddress address = new OzoneAddress(str);
+ address.ensureVolumeAddress();
+ return address;
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/package-info.java
similarity index 88%
copy from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/package-info.java
copy to hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/package-info.java
index c344c35..d5cdc9b 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/package-info.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/package-info.java
@@ -18,6 +18,6 @@
*/
/**
- * Netty-based HTTP server implementation for Ozone.
+ * Commands for Ozone volumes.
*/
-package org.apache.hadoop.ozone.web.ozShell.bucket;
+package org.apache.hadoop.ozone.shell.volume;
\ No newline at end of file
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/TestOzoneAddress.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneAddress.java
similarity index 98%
rename from hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/TestOzoneAddress.java
rename to hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneAddress.java
index 7ae0520..3ab866c 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/TestOzoneAddress.java
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneAddress.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hadoop.ozone.web.ozShell;
+package org.apache.hadoop.ozone.shell;
import java.io.IOException;
import java.util.Arrays;
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/package-info.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/package-info.java
similarity index 94%
rename from hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/package-info.java
rename to hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/package-info.java
index 80c1985..a7c89d8 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/package-info.java
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/package-info.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.ozone.web.ozShell;
+package org.apache.hadoop.ozone.shell;
/**
* Tests for ozone shell..
*/
diff --git a/hadoop-ozone/upgrade/pom.xml b/hadoop-ozone/upgrade/pom.xml
index 90739a0..ed80d9c 100644
--- a/hadoop-ozone/upgrade/pom.xml
+++ b/hadoop-ozone/upgrade/pom.xml
@@ -31,6 +31,11 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-test-utils</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-common</artifactId>
</dependency>
<dependency>
diff --git a/pom.xml b/pom.xml
index a06830e..8c697d7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -69,7 +69,7 @@
</organization>
<properties>
- <hadoop.version>3.2.0</hadoop.version>
+ <hadoop.version>3.2.1</hadoop.version>
<!-- version for hdds/ozone components -->
<hdds.version>${ozone.version}</hdds.version>
@@ -78,7 +78,7 @@
<declared.ozone.version>${ozone.version}</declared.ozone.version>
<!-- Apache Ratis version -->
- <ratis.version>0.6.0-a320ae0-SNAPSHOT</ratis.version>
+ <ratis.version>0.6.0-cac3336-SNAPSHOT</ratis.version>
<distMgmtSnapshotsId>apache.snapshots.https</distMgmtSnapshotsId>
<distMgmtSnapshotsName>Apache Development Snapshot Repository</distMgmtSnapshotsName>
<distMgmtSnapshotsUrl>https://repository.apache.org/content/repositories/snapshots</distMgmtSnapshotsUrl>
@@ -118,27 +118,33 @@
<java.security.egd>file:///dev/urandom</java.security.egd>
- <!-- avro version -->
- <avro.version>1.7.7</avro.version>
-
<bouncycastle.version>1.60</bouncycastle.version>
<!-- jersey version -->
<jersey.version>1.19</jersey.version>
+ <jersey2.version>2.27</jersey2.version>
<!-- jackson versions -->
- <jackson.version>1.9.13</jackson.version>
<jackson2.version>2.10.3</jackson2.version>
<!-- jaegertracing veresion -->
- <jaeger.version>0.34.0</jaeger.version>
+ <jaeger.version>1.2.0</jaeger.version>
+ <opentracing.version>0.33.0</opentracing.version>
+
+ <jmh.version>1.19</jmh.version>
+ <hk2.version>2.5.0</hk2.version>
<!-- httpcomponents versions -->
<httpclient.version>4.5.2</httpclient.version>
<httpcore.version>4.4.4</httpcore.version>
- <!-- SLF4J version -->
+ <!-- SLF4J/LOG4J version -->
<slf4j.version>1.7.25</slf4j.version>
+ <log4j.version>1.2.17</log4j.version>
+ <log4j2.version>2.11.0</log4j2.version>
+ <disruptor.version>3.4.2</disruptor.version>
+
+ <prometheus.version>0.7.0</prometheus.version>
<!-- com.google.re2j version -->
<re2j.version>1.1</re2j.version>
@@ -146,16 +152,13 @@
<!-- ProtocolBuffer version, used to verify the protoc version and -->
<!-- define the protobuf JAR version -->
<protobuf.version>2.5.0</protobuf.version>
- <protoc.path>${env.HADOOP_PROTOC_PATH}</protoc.path>
- <curator.version>2.12.0</curator.version>
<findbugs.version>3.0.0</findbugs.version>
<spotbugs.version>3.1.12</spotbugs.version>
<dnsjava.version>2.1.7</dnsjava.version>
<guava.version>28.2-jre</guava.version>
<guice.version>4.0</guice.version>
- <joda-time.version>2.9.9</joda-time.version>
<!-- Required for testing LDAP integration -->
<apacheds.version>2.0.0-M21</apacheds.version>
@@ -166,14 +169,22 @@
<hikari.version>2.4.12</hikari.version>
<mssql.version>6.2.1.jre7</mssql.version>
<okhttp.version>2.7.5</okhttp.version>
- <mockito-core.version>2.28.2</mockito-core.version>
+ <mockito1-hadoop.version>1.8.5</mockito1-hadoop.version>
+ <mockito1-powermock.version>1.10.19</mockito1-powermock.version>
+ <mockito2.version>2.28.2</mockito2.version>
+ <hamcrest.version>1.3</hamcrest.version>
+ <powermock1.version>1.6.5</powermock1.version>
+ <powermock2.version>2.0.4</powermock2.version>
+ <junit.jupiter.version>5.3.1</junit.jupiter.version>
<!-- Maven protoc compiler -->
<protobuf-maven-plugin.version>0.5.1</protobuf-maven-plugin.version>
- <datanode.protobuf-compile.version>3.10.0</datanode.protobuf-compile.version>
- <datanode.grpc-compile.version>1.24.0</datanode.grpc-compile.version>
+ <protobuf-compile.version>3.11.0</protobuf-compile.version>
+ <grpc-compile.version>1.28.1</grpc-compile.version>
<os-maven-plugin.version>1.5.0.Final</os-maven-plugin.version>
+ <netty.version>4.1.48.Final</netty.version>
+
<!-- define the Java language version used by the compiler -->
<javac.version>1.8</javac.version>
@@ -194,7 +205,7 @@
<maven-clean-plugin.version>2.5</maven-clean-plugin.version>
<maven-compiler-plugin.version>3.1</maven-compiler-plugin.version>
<maven-install-plugin.version>2.5.1</maven-install-plugin.version>
- <maven-resources-plugin.version>2.6</maven-resources-plugin.version>
+ <maven-resources-plugin.version>3.1.0</maven-resources-plugin.version>
<maven-shade-plugin.version>3.2.0</maven-shade-plugin.version>
<maven-jar-plugin.version>2.5</maven-jar-plugin.version>
<maven-war-plugin.version>3.1.0</maven-war-plugin.version>
@@ -214,12 +225,13 @@
<maven-checkstyle-plugin.version>3.0.0</maven-checkstyle-plugin.version>
<checkstyle.version>8.19</checkstyle.version>
<surefire.fork.timeout>900</surefire.fork.timeout>
- <aws-java-sdk.version>1.11.375</aws-java-sdk.version>
+ <aws-java-sdk.version>1.11.615</aws-java-sdk.version>
<hsqldb.version>2.3.4</hsqldb.version>
- <frontend-maven-plugin.version>1.5</frontend-maven-plugin.version>
+ <frontend-maven-plugin.version>1.6</frontend-maven-plugin.version>
<!-- the version of Hadoop declared in the version resources; can be overridden
so that Hadoop 3.x can declare itself a 2.x artifact. -->
<declared.hadoop.version>${hadoop.version}</declared.hadoop.version>
+ <proto-backwards-compatibility.version>1.0.5</proto-backwards-compatibility.version>
<swagger-annotations-version>1.5.4</swagger-annotations-version>
<snakeyaml.version>1.16</snakeyaml.version>
@@ -241,6 +253,11 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>info.picocli</groupId>
+ <artifactId>picocli</artifactId>
+ <version>3.9.6</version>
+ </dependency>
+ <dependency>
<groupId>jdiff</groupId>
<artifactId>jdiff</artifactId>
<version>${jdiff.version}</version>
@@ -646,12 +663,12 @@
<dependency>
<groupId>org.openjdk.jmh</groupId>
<artifactId>jmh-core</artifactId>
- <version>1.19</version>
+ <version>${jmh.version}</version>
</dependency>
<dependency>
<groupId>org.openjdk.jmh</groupId>
<artifactId>jmh-generator-annprocess</artifactId>
- <version>1.19</version>
+ <version>${jmh.version}</version>
</dependency>
@@ -718,6 +735,31 @@
<version>3.6</version>
</dependency>
<dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-pool2</artifactId>
+ <version>2.6.0</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-validator</groupId>
+ <artifactId>commons-validator</artifactId>
+ <version>1.6</version>
+ </dependency>
+ <dependency>
+ <groupId>javax.activation</groupId>
+ <artifactId>activation</artifactId>
+ <version>1.1.1</version>
+ </dependency>
+ <dependency>
+ <groupId>javax.annotation</groupId>
+ <artifactId>javax.annotation-api</artifactId>
+ <version>1.2</version>
+ </dependency>
+ <dependency>
+ <groupId>javax.enterprise</groupId>
+ <artifactId>cdi-api</artifactId>
+ <version>1.2</version>
+ </dependency>
+ <dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
<version>3.1.0</version>
@@ -768,6 +810,67 @@
<artifactId>javax.servlet</artifactId>
<version>3.1</version>
</dependency>
+ <dependency>
+ <groupId>org.glassfish.hk2</groupId>
+ <artifactId>guice-bridge</artifactId>
+ <version>${hk2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.hk2</groupId>
+ <artifactId>hk2-api</artifactId>
+ <version>${hk2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.containers</groupId>
+ <artifactId>jersey-container-servlet</artifactId>
+ <version>${jersey2.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.glassfish.hk2</groupId>
+ <artifactId>hk2-api</artifactId>
+ </exclusion>
+ </exclusions>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.containers</groupId>
+ <artifactId>jersey-container-servlet-core</artifactId>
+ <version>${jersey2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.core</groupId>
+ <artifactId>jersey-server</artifactId>
+ <version>${jersey2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.ext.cdi</groupId>
+ <artifactId>jersey-cdi1x</artifactId>
+ <version>${jersey2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.inject</groupId>
+ <artifactId>jersey-hk2</artifactId>
+ <version>${jersey2.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.glassfish.hk2</groupId>
+ <artifactId>hk2-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.glassfish.hk2</groupId>
+ <artifactId>hk2-utils</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.glassfish.hk2.external</groupId>
+ <artifactId>aopalliance-repackaged</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.media</groupId>
+ <artifactId>jersey-media-json-jackson</artifactId>
+ <version>${jersey2.version}</version>
+ </dependency>
<dependency>
<groupId>org.codehaus.plexus</groupId>
@@ -818,10 +921,37 @@
</dependency>
<dependency>
+ <groupId>com.google.errorprone</groupId>
+ <artifactId>error_prone_annotations</artifactId>
+ <version>2.2.0</version>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
<version>${guice.version}</version>
</dependency>
+ <dependency>
+ <groupId>com.google.inject.extensions</groupId>
+ <artifactId>guice-assistedinject</artifactId>
+ <version>${guice.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.inject.extensions</groupId>
+ <artifactId>guice-multibindings</artifactId>
+ <version>${guice.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.inject.extensions</groupId>
+ <artifactId>guice-servlet</artifactId>
+ <version>${guice.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.jolbox</groupId>
+ <artifactId>bonecp</artifactId>
+ <version>0.8.0.RELEASE</version>
+ </dependency>
<dependency>
<groupId>cglib</groupId>
@@ -830,12 +960,6 @@
</dependency>
<dependency>
- <groupId>com.google.inject.extensions</groupId>
- <artifactId>guice-servlet</artifactId>
- <version>${guice.version}</version>
- </dependency>
-
- <dependency>
<groupId>com.sun.jersey.contribs</groupId>
<artifactId>jersey-guice</artifactId>
<version>${jersey.version}</version>
@@ -895,6 +1019,11 @@
<groupId>org.apache.ratis</groupId>
<version>${ratis.version}</version>
</dependency>
+ <dependency>
+ <artifactId>ratis-tools</artifactId>
+ <groupId>org.apache.ratis</groupId>
+ <version>${ratis.version}</version>
+ </dependency>
<dependency>
<groupId>io.netty</groupId>
@@ -905,7 +1034,7 @@
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
- <version>4.1.47.Final</version>
+ <version>${netty.version}</version>
</dependency>
<dependency>
@@ -922,7 +1051,7 @@
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
- <version>1.1.3</version>
+ <version>1.2</version>
<exclusions>
<exclusion>
<groupId>avalon-framework</groupId>
@@ -946,7 +1075,7 @@
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
- <version>1.2.17</version>
+ <version>${log4j.version}</version>
<exclusions>
<exclusion>
<groupId>com.sun.jdmk</groupId>
@@ -972,14 +1101,13 @@
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-bundle</artifactId>
+ <artifactId>aws-java-sdk-core</artifactId>
<version>${aws-java-sdk.version}</version>
- <exclusions>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-s3</artifactId>
+ <version>${aws-java-sdk.version}</version>
</dependency>
<dependency>
<groupId>org.apache.mina</groupId>
@@ -1012,6 +1140,12 @@
<version>4.11</version>
</dependency>
<dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ <version>${junit.jupiter.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<version>3.2.2</version>
@@ -1073,26 +1207,6 @@
<version>5.0.3</version>
</dependency>
<dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-mapper-asl</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-core-asl</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-jaxrs</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-xc</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>${jackson2.version}</version>
@@ -1118,14 +1232,37 @@
<version>${jackson2.version}</version>
</dependency>
<dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>1.8.5</version>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-xml</artifactId>
+ <version>${jackson2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.datatype</groupId>
+ <artifactId>jackson-datatype-jsr310</artifactId>
+ <version>${jackson2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ <version>${hamcrest.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-core</artifactId>
+ <version>${hamcrest.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.jmockit</groupId>
+ <artifactId>jmockit</artifactId>
+ <version>1.24</version>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
- <version>${mockito-core.version}</version>
+ <version>${mockito2.version}</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -1145,11 +1282,6 @@
</exclusions>
</dependency>
<dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- <version>${avro.version}</version>
- </dependency>
- <dependency>
<groupId>net.sf.kosmosfs</groupId>
<artifactId>kfs</artifactId>
<version>0.3</version>
@@ -1201,6 +1333,26 @@
<version>3.2.4</version>
</dependency>
<dependency>
+ <groupId>io.jaegertracing</groupId>
+ <artifactId>jaeger-client</artifactId>
+ <version>${jaeger.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.opentracing</groupId>
+ <artifactId>opentracing-util</artifactId>
+ <version>${opentracing.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.prometheus</groupId>
+ <artifactId>simpleclient_dropwizard</artifactId>
+ <version>${prometheus.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.prometheus</groupId>
+ <artifactId>simpleclient_common</artifactId>
+ <version>${prometheus.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-sls</artifactId>
<version>${hadoop.version}</version>
@@ -1217,14 +1369,30 @@
<scope>provided</scope>
</dependency>
<dependency>
+ <groupId>com.github.spotbugs</groupId>
+ <artifactId>spotbugs-annotations</artifactId>
+ <version>${spotbugs.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
<version>${findbugs.version}</version>
</dependency>
<dependency>
+ <groupId>com.sun.xml.bind</groupId>
+ <artifactId>jaxb-impl</artifactId>
+ <version>2.3.0.1</version>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.xml.bind</groupId>
+ <artifactId>jaxb-core</artifactId>
+ <version>2.3.0.1</version>
+ </dependency>
+ <dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
- <version>2.2.11</version>
+ <version>2.3.0</version>
</dependency>
<dependency>
<groupId>org.codehaus.jettison</groupId>
@@ -1301,26 +1469,6 @@
</dependency>
<dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-recipes</artifactId>
- <version>${curator.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-client</artifactId>
- <version>${curator.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
- <version>${curator.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-test</artifactId>
- <version>${curator.version}</version>
- </dependency>
- <dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk16</artifactId>
<version>${bouncycastle.version}</version>
@@ -1328,12 +1476,6 @@
</dependency>
<dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- <version>${joda-time.version}</version>
- </dependency>
-
- <dependency>
<groupId>com.nimbusds</groupId>
<artifactId>nimbus-jose-jwt</artifactId>
<version>7.9</version>
@@ -1413,17 +1555,77 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.jboss.weld.servlet</groupId>
+ <artifactId>weld-servlet</artifactId>
+ <version>2.4.7.Final</version>
+ </dependency>
+ <dependency>
<groupId>org.jruby.jcodings</groupId>
<artifactId>jcodings</artifactId>
<version>1.0.13</version>
</dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>${powermock2.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito</artifactId>
+ <version>${powermock1.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito2</artifactId>
+ <version>${powermock2.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.rocksdb</groupId>
+ <artifactId>rocksdbjni</artifactId>
+ <version>6.6.4</version>
+ </dependency>
+ <dependency>
+ <groupId>org.xerial</groupId>
+ <artifactId>sqlite-jdbc</artifactId>
+ <version>3.25.2</version>
+ </dependency>
</dependencies>
</dependencyManagement>
<build>
+ <extensions>
+ <extension>
+ <groupId>kr.motd.maven</groupId>
+ <artifactId>os-maven-plugin</artifactId>
+ <version>${os-maven-plugin.version}</version>
+ </extension>
+ </extensions>
<pluginManagement>
<plugins>
<plugin>
+ <groupId>com.salesforce.servicelibs</groupId>
+ <artifactId>proto-backwards-compatibility</artifactId>
+ <version>${proto-backwards-compatibility.version}</version>
+ <configuration>
+ <protoSourceRoot>${basedir}/src/main/proto/</protoSourceRoot>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>backwards-compatibility-check</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>com.github.eirslett</groupId>
+ <artifactId>frontend-maven-plugin</artifactId>
+ <version>${frontend-maven-plugin.version}</version>
+ </plugin>
+ <plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>${build-helper-maven-plugin.version}</version>
@@ -1534,11 +1736,6 @@
<version>${maven-source-plugin.version}</version>
</plugin>
<plugin>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-maven-plugin</artifactId>
- <version>${avro.version}</version>
- </plugin>
- <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<version>${maven-resources-plugin.version}</version>
@@ -1598,6 +1795,11 @@
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.3</version>
</plugin>
+ <plugin>
+ <groupId>io.fabric8</groupId>
+ <artifactId>docker-maven-plugin</artifactId>
+ <version>0.29.0</version>
+ </plugin>
</plugins>
</pluginManagement>
